diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 344ac8e91..35fc74464 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,22 +1,10 @@ - -Please make sure you keep the title of your pull request short and informative, -and that you fill in the following template accurately (don't forget to remove -the fields that you do not use and the example texts!). You can also add relevant labels in the right -sidebar. +## Status + ---> +Ready to merge -## Pull request information +## Description + -- **Status**: ? (ready to merge / in development) -- **Kind of changes**: ? (bug fix / new feature / documentation...) -- **Related PR or issue**: ? (optional) - -## Description - - - - -? (general description) diff --git a/.gitignore b/.gitignore index 29d788c27..7f2603e8d 100644 --- a/.gitignore +++ b/.gitignore @@ -111,3 +111,8 @@ flow.ang *.ang.old *.sang +# local configuration file for data pipeline +**/data_pipeline_config + +temp.pkl + diff --git a/.travis.yml b/.travis.yml index 297281bc7..a802f5d23 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,16 +38,16 @@ before_install: # Set up requirements for flow - conda env create -f environment.yml - - source activate flow + - source activate circles # [sumo] dependencies and binaries - - pushd $HOME/build/flow-project + - pushd $HOME/build/CIRCLES-consortium - ./flow/scripts/setup_sumo_ubuntu1604.sh - popd - source ~/.bashrc # [aimsun] install the conda env and update the path to the env - - pushd $HOME/build/flow-project + - pushd $HOME/build/CIRCLES-consortium - ./flow/scripts/setup_aimsun.sh - popd - source ~/.bashrc diff --git a/docs/Prius_EnergyModel.pdf b/docs/Prius_EnergyModel.pdf new file mode 100644 index 000000000..4884fb5cc Binary files /dev/null and b/docs/Prius_EnergyModel.pdf differ diff --git a/docs/Tacoma_EnergyModel.pdf b/docs/Tacoma_EnergyModel.pdf new file mode 100644 index 000000000..8fe16d05f Binary files /dev/null and b/docs/Tacoma_EnergyModel.pdf differ diff --git a/environment.yml b/environment.yml index 97d9ad6f8..ee6e612d0 100644 --- a/environment.yml +++ b/environment.yml @@ -1,4 +1,4 @@ -name: flow +name: circles dependencies: - python==3.7.3 @@ -18,6 +18,7 @@ dependencies: - cached_property - joblib==0.10.3 - matplotlib==3.0.0 + - pandas==1.1.2 - dill - lz4 - ray==0.8.0 @@ -26,3 +27,5 @@ dependencies: - opencv-python - boto3==1.10.45 - redis~=2.10.6 + - tabulate + - pytz diff --git a/examples/README.md b/examples/README.md index a9d681131..8156831fe 100644 --- a/examples/README.md +++ b/examples/README.md @@ -53,11 +53,11 @@ trained through RL algorithms provided by *RLlib*. To execute these examples, run ```shell script -python train.py EXP_CONFIG --rl_trainer "rllib" +python train.py EXP_CONFIG --rl_trainer "rllib" --algorithm ``` where `EXP_CONFIG` is the name of the experiment configuration file, as located -in `exp_configs/rl/singleagent` or `exp_configs/rl/multiagent.` - +in `exp_configs/rl/singleagent` or `exp_configs/rl/multiagent.` Here `` +should be the name of your desired algorithm. Currently we support PPO and TD3. ### stable-baselines diff --git a/examples/checkpoints/multiagent_i210/checkpoint_200/checkpoint-200 b/examples/checkpoints/multiagent_i210/checkpoint_200/checkpoint-200 new file mode 100644 index 000000000..a4bf4adb5 Binary files /dev/null and b/examples/checkpoints/multiagent_i210/checkpoint_200/checkpoint-200 differ diff --git a/examples/checkpoints/multiagent_i210/checkpoint_200/checkpoint-200.tune_metadata b/examples/checkpoints/multiagent_i210/checkpoint_200/checkpoint-200.tune_metadata new file mode 100644 index 000000000..cb7042b38 Binary files /dev/null and b/examples/checkpoints/multiagent_i210/checkpoint_200/checkpoint-200.tune_metadata differ diff --git a/examples/checkpoints/multiagent_i210/events.out.tfevents.1599012086.ip-172-31-29-2 b/examples/checkpoints/multiagent_i210/events.out.tfevents.1599012086.ip-172-31-29-2 new file mode 100644 index 000000000..f0e36151e Binary files /dev/null and b/examples/checkpoints/multiagent_i210/events.out.tfevents.1599012086.ip-172-31-29-2 differ diff --git a/examples/checkpoints/multiagent_i210/params.json b/examples/checkpoints/multiagent_i210/params.json new file mode 100644 index 000000000..bbe2c46f3 --- /dev/null +++ b/examples/checkpoints/multiagent_i210/params.json @@ -0,0 +1,139 @@ +{ + "batch_mode": "truncate_episodes", + "callbacks": { + "on_episode_end": ".on_episode_end at 0x7f3b000ef0d0>", + "on_episode_start": ".on_episode_start at 0x7f3b004ed620>", + "on_episode_step": ".on_episode_step at 0x7f3b000ef048>", + "on_train_result": ".on_train_result at 0x7f3b000ef158>" + }, + "clip_actions": true, + "clip_param": 0.3, + "clip_rewards": null, + "collect_metrics_timeout": 180, + "compress_observations": false, + "custom_resources_per_worker": {}, + "eager": false, + "eager_tracing": false, + "entropy_coeff": 0.0, + "entropy_coeff_schedule": null, + "env": "I210MultiEnv-v2", + "env_config": { + "flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", + "run": "" + }, + "evaluation_config": {}, + "evaluation_interval": null, + "evaluation_num_episodes": 10, + "gamma": 0.995, + "grad_clip": null, + "horizon": 1000, + "ignore_worker_failures": false, + "input": "sampler", + "input_evaluation": [ + "is", + "wis" + ], + "kl_coeff": 0.2, + "kl_target": 0.02, + "lambda": 0.97, + "local_tf_session_args": { + "inter_op_parallelism_threads": 8, + "intra_op_parallelism_threads": 8 + }, + "log_level": "WARN", + "log_sys_usage": true, + "lr": 0.0005, + "lr_schedule": null, + "memory": 0, + "memory_per_worker": 0, + "metrics_smoothing_episodes": 100, + "min_iter_time_s": 0, + "model": { + "conv_activation": "relu", + "conv_filters": null, + "custom_action_dist": null, + "custom_model": null, + "custom_options": {}, + "custom_preprocessor": null, + "dim": 84, + "fcnet_activation": "tanh", + "fcnet_hiddens": [ + 32, + 32 + ], + "framestack": true, + "free_log_std": false, + "grayscale": false, + "lstm_cell_size": 256, + "lstm_use_prev_action_reward": false, + "max_seq_len": 20, + "no_final_linear": false, + "state_shape": null, + "use_lstm": false, + "vf_share_layers": true, + "zero_mean": true + }, + "monitor": false, + "multiagent": { + "policies": { + "av": [ + null, + "Box(5,)", + "Box(1,)", + {} + ] + }, + "policies_to_train": null, + "policy_mapping_fn": "" + }, + "no_done_at_end": true, + "no_eager_on_workers": false, + "num_cpus_for_driver": 1, + "num_cpus_per_worker": 1, + "num_envs_per_worker": 1, + "num_gpus": 0, + "num_gpus_per_worker": 0, + "num_sgd_iter": 10, + "num_workers": 17, + "object_store_memory": 0, + "object_store_memory_per_worker": 0, + "observation_filter": "NoFilter", + "optimizer": {}, + "output": null, + "output_compress_columns": [ + "obs", + "new_obs" + ], + "output_max_file_size": 67108864, + "postprocess_inputs": false, + "preprocessor_pref": "deepmind", + "remote_env_batch_wait_ms": 0, + "remote_worker_envs": false, + "sample_async": false, + "sample_batch_size": 200, + "seed": null, + "sgd_minibatch_size": 128, + "shuffle_buffer_size": 0, + "shuffle_sequences": true, + "simple_optimizer": false, + "soft_horizon": false, + "synchronize_filters": true, + "tf_session_args": { + "allow_soft_placement": true, + "device_count": { + "CPU": 1 + }, + "gpu_options": { + "allow_growth": true + }, + "inter_op_parallelism_threads": 2, + "intra_op_parallelism_threads": 2, + "log_device_placement": false + }, + "timesteps_per_iteration": 0, + "train_batch_size": 17000, + "use_gae": true, + "vf_clip_param": 10.0, + "vf_loss_coeff": 1.0, + "vf_share_layers": false +} \ No newline at end of file diff --git a/examples/checkpoints/multiagent_i210/params.pkl b/examples/checkpoints/multiagent_i210/params.pkl new file mode 100644 index 000000000..e1d7ecb28 Binary files /dev/null and b/examples/checkpoints/multiagent_i210/params.pkl differ diff --git a/examples/checkpoints/multiagent_i210/result.json b/examples/checkpoints/multiagent_i210/result.json new file mode 100644 index 000000000..0320b5ccc --- /dev/null +++ b/examples/checkpoints/multiagent_i210/result.json @@ -0,0 +1,200 @@ +{"episode_reward_max": 53536.21766956626, "episode_reward_min": 49824.838241618454, "episode_reward_mean": 50991.80364016202, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 491.2057597088948}, "policy_reward_mean": {"av": 258.45577277363}, "custom_metrics": {"avg_speed_mean": 3.465768793213715, "avg_speed_min": 3.1503079652604264, "avg_speed_max": 3.7000405944354635, "avg_speed_avs_mean": 2.421615314297735, "avg_speed_avs_min": 2.2852643750408235, "avg_speed_avs_max": 2.5357745052694, "avg_headway_avs_mean": 29.426361406406777, "avg_headway_avs_min": 24.87713561650247, "avg_headway_avs_max": 31.584700626121524, "std_headway_avs_mean": 44.566518885620106, "std_headway_avs_min": 38.07422953101362, "std_headway_avs_max": 49.986159831576195, "avg_accel_avs_mean": 0.6515949334755353, "avg_accel_avs_min": 0.6452753112046652, "avg_accel_avs_max": 0.65660394406502, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 14.351731980502981, "avg_mpg_per_veh_min": 14.022551226866414, "avg_mpg_per_veh_max": 14.678953214977494, "num_cars_mean": 829.4388552623848, "num_cars_min": 796.061938061938, "num_cars_max": 886.7102897102897, "avg_mpg_mean": 14.348825610423404, "avg_mpg_min": 14.0052739234155, "avg_mpg_max": 14.681479259773928}, "sampler_perf": {"mean_env_wait_ms": 1951.5466271426399, "mean_processing_ms": 435.9866115820914, "mean_inference_ms": 2.068925087408642}, "off_policy_estimator": {}, "info": {"num_steps_trained": 1269120, "num_steps_sampled": 17000, "sample_time_ms": 2478411.125, "load_time_ms": 718.608, "grad_time_ms": 220658.43, "update_time_ms": 533.098, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 175.87734985351562, "policy_loss": -0.0178261399269104, "vf_loss": 175.89517211914062, "vf_explained_var": 0.051522742956876755, "vf_preds": 15.201702117919922, "kl": 0.03760787472128868, "entropy": 1.41536283493042, "entropy_coeff": 0.0, "advantages": -2.2550771973328665e-05, "rewards": 0.6495282649993896}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 17000, "episodes_total": 17, "training_iteration": 1, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_02-01-26", "timestamp": 1599012086, "time_this_iter_s": 2700.7213637828827, "time_total_s": 2700.7213637828827, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc75fbf8>", "on_episode_step": ".on_episode_step at 0x7f3afc75fc80>", "on_episode_end": ".on_episode_end at 0x7f3afc75fd08>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc75fd90>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 2700.7213637828827, "timesteps_since_restore": 17000, "iterations_since_restore": 1, "perf": {"cpu_util_percent": 89.34582791785807, "ram_util_percent": 27.292955549779045}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 107464.39266329867, "episode_reward_min": 49824.838241618454, "episode_reward_mean": 77455.66596329483, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 573.698932479357}, "policy_reward_mean": {"av": 329.84627288978277}, "custom_metrics": {"avg_speed_mean": 4.867773873632423, "avg_speed_min": 3.1503079652604264, "avg_speed_max": 7.0043220707592955, "avg_speed_avs_mean": 4.0385885913652535, "avg_speed_avs_min": 2.2852643750408235, "avg_speed_avs_max": 6.427311805030021, "avg_headway_avs_mean": 28.12107947179324, "avg_headway_avs_min": 24.876774807121446, "avg_headway_avs_max": 31.584700626121524, "std_headway_avs_mean": 39.64843703530233, "std_headway_avs_min": 31.97683797457919, "std_headway_avs_max": 49.986159831576195, "avg_accel_avs_mean": 0.6976829407181869, "avg_accel_avs_min": 0.6452753112046652, "avg_accel_avs_max": 0.7502014433796591, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 17.10794291047819, "avg_mpg_per_veh_min": 14.022551226866414, "avg_mpg_per_veh_max": 20.779538032540017, "num_cars_mean": 786.2164864547218, "num_cars_min": 707.898, "num_cars_max": 886.7102897102897, "avg_mpg_mean": 17.05492579334615, "avg_mpg_min": 14.0052739234155, "avg_mpg_max": 20.55764396115954}, "sampler_perf": {"mean_env_wait_ms": 1799.8147810612677, "mean_processing_ms": 432.30573086205146, "mean_inference_ms": 2.024747971918002}, "off_policy_estimator": {}, "info": {"num_steps_trained": 2231680, "num_steps_sampled": 34000, "sample_time_ms": 2142503.237, "load_time_ms": 526.427, "grad_time_ms": 194097.931, "update_time_ms": 270.549, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 1515.5919189453125, "policy_loss": -0.006165079306811094, "vf_loss": 1515.59814453125, "vf_explained_var": 0.01981254294514656, "vf_preds": 46.72843933105469, "kl": 0.021103164181113243, "entropy": 1.355449914932251, "entropy_coeff": 0.0, "advantages": 2.408049113000743e-05, "rewards": 1.7663452625274658}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 34000, "episodes_total": 34, "training_iteration": 2, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_02-34-21", "timestamp": 1599014061, "time_this_iter_s": 1974.8316853046417, "time_total_s": 4675.553049087524, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc75fb70>", "on_episode_step": ".on_episode_step at 0x7f3afc75f7b8>", "on_episode_end": ".on_episode_end at 0x7f3afc75fae8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc75fa60>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 4675.553049087524, "timesteps_since_restore": 34000, "iterations_since_restore": 2, "perf": {"cpu_util_percent": 88.5551368645574, "ram_util_percent": 28.905865623889085}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 118265.05449032258, "episode_reward_min": 49824.838241618454, "episode_reward_mean": 90345.16516889137, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 573.698932479357}, "policy_reward_mean": {"av": 351.85974979866063}, "custom_metrics": {"avg_speed_mean": 5.137939401687872, "avg_speed_min": 3.1503079652604264, "avg_speed_max": 7.0043220707592955, "avg_speed_avs_mean": 4.5384375988391605, "avg_speed_avs_min": 2.2852643750408235, "avg_speed_avs_max": 6.427311805030021, "avg_headway_avs_mean": 23.580569776910124, "avg_headway_avs_min": 13.820957272018196, "avg_headway_avs_max": 31.584700626121524, "std_headway_avs_mean": 32.845030944595436, "std_headway_avs_min": 17.697461254737625, "std_headway_avs_max": 49.986159831576195, "avg_accel_avs_mean": 0.7190814607273331, "avg_accel_avs_min": 0.6452753112046652, "avg_accel_avs_max": 0.7693432735781432, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 17.82717285976747, "avg_mpg_per_veh_min": 14.022551226866414, "avg_mpg_per_veh_max": 20.779538032540017, "num_cars_mean": 798.3105792051086, "num_cars_min": 707.898, "num_cars_max": 886.7102897102897, "avg_mpg_mean": 17.776661342019242, "avg_mpg_min": 14.0052739234155, "avg_mpg_max": 20.55764396115954}, "sampler_perf": {"mean_env_wait_ms": 1731.9095648453688, "mean_processing_ms": 431.90248524135, "mean_inference_ms": 2.003662268762457}, "off_policy_estimator": {}, "info": {"num_steps_trained": 3303168, "num_steps_sampled": 51000, "sample_time_ms": 2077455.513, "load_time_ms": 473.522, "grad_time_ms": 192762.287, "update_time_ms": 182.948, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 1246.4703369140625, "policy_loss": -0.0020281535107642412, "vf_loss": 1246.4722900390625, "vf_explained_var": 0.01744106039404869, "vf_preds": 53.53205871582031, "kl": 0.010345520451664925, "entropy": 1.334789514541626, "entropy_coeff": 0.0, "advantages": -2.6600930596032413e-06, "rewards": 1.7726927995681763}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 51000, "episodes_total": 51, "training_iteration": 3, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_03-09-59", "timestamp": 1599016199, "time_this_iter_s": 2138.184633255005, "time_total_s": 6813.737682342529, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc71eea0>", "on_episode_step": ".on_episode_step at 0x7f3afc71ef28>", "on_episode_end": ".on_episode_end at 0x7f3afc6d2048>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc6d20d0>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 6813.737682342529, "timesteps_since_restore": 51000, "iterations_since_restore": 3, "perf": {"cpu_util_percent": 90.5456336178595, "ram_util_percent": 29.167826657912016}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 118919.30838240229, "episode_reward_min": 49824.838241618454, "episode_reward_mean": 97278.69234076979, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 573.698932479357}, "policy_reward_mean": {"av": 361.4529850375578}, "custom_metrics": {"avg_speed_mean": 5.233432251588009, "avg_speed_min": 3.1503079652604264, "avg_speed_max": 7.0043220707592955, "avg_speed_avs_mean": 4.766056331089597, "avg_speed_avs_min": 2.2852643750408235, "avg_speed_avs_max": 6.427311805030021, "avg_headway_avs_mean": 20.607955824497104, "avg_headway_avs_min": 11.054985870000891, "avg_headway_avs_max": 31.584700626121524, "std_headway_avs_mean": 28.426249188358874, "std_headway_avs_min": 13.774963041308256, "std_headway_avs_max": 49.986159831576195, "avg_accel_avs_mean": 0.72985107569687, "avg_accel_avs_min": 0.6452753112046652, "avg_accel_avs_max": 0.7713943516542706, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 18.08799752361908, "avg_mpg_per_veh_min": 14.022551226866414, "avg_mpg_per_veh_max": 20.779538032540017, "num_cars_mean": 809.7468902861843, "num_cars_min": 707.898, "num_cars_max": 886.7102897102897, "avg_mpg_mean": 18.03975999331161, "avg_mpg_min": 14.0052739234155, "avg_mpg_max": 20.55764396115954}, "sampler_perf": {"mean_env_wait_ms": 1694.0110322384285, "mean_processing_ms": 431.3977959346236, "mean_inference_ms": 1.9908475804172079}, "off_policy_estimator": {}, "info": {"num_steps_trained": 4409088, "num_steps_sampled": 68000, "sample_time_ms": 2055700.54, "load_time_ms": 454.643, "grad_time_ms": 191314.12, "update_time_ms": 139.236, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 1116.083740234375, "policy_loss": -0.0013218637323006988, "vf_loss": 1116.0850830078125, "vf_explained_var": 0.011604256927967072, "vf_preds": 60.268943786621094, "kl": 0.00776265561580658, "entropy": 1.328734278678894, "entropy_coeff": 0.0, "advantages": 9.374762157676741e-07, "rewards": 1.7461309432983398}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 68000, "episodes_total": 68, "training_iteration": 4, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_03-46-17", "timestamp": 1599018377, "time_this_iter_s": 2178.2033693790436, "time_total_s": 8991.941051721573, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc75fa60>", "on_episode_step": ".on_episode_step at 0x7f3afc75fd90>", "on_episode_end": ".on_episode_end at 0x7f3afc75fbf8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc75f268>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 8991.941051721573, "timesteps_since_restore": 68000, "iterations_since_restore": 4, "perf": {"cpu_util_percent": 90.38613797549968, "ram_util_percent": 29.30789813023856}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 120501.62598433408, "episode_reward_min": 49824.838241618454, "episode_reward_mean": 101468.52283203852, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 573.698932479357}, "policy_reward_mean": {"av": 366.0325272980213}, "custom_metrics": {"avg_speed_mean": 5.279436697289004, "avg_speed_min": 3.1503079652604264, "avg_speed_max": 7.0043220707592955, "avg_speed_avs_mean": 4.897007114762875, "avg_speed_avs_min": 2.2852643750408235, "avg_speed_avs_max": 6.427311805030021, "avg_headway_avs_mean": 18.496891182667103, "avg_headway_avs_min": 9.508620418375772, "avg_headway_avs_max": 31.584700626121524, "std_headway_avs_mean": 25.26788963141113, "std_headway_avs_min": 11.525565603055359, "std_headway_avs_max": 49.986159831576195, "avg_accel_avs_mean": 0.7363061989813814, "avg_accel_avs_min": 0.6452753112046652, "avg_accel_avs_max": 0.7748883491456845, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 18.12716865621916, "avg_mpg_per_veh_min": 14.022551226866414, "avg_mpg_per_veh_max": 20.779538032540017, "num_cars_mean": 818.4534769348297, "num_cars_min": 707.898, "num_cars_max": 886.7102897102897, "avg_mpg_mean": 18.082682988512005, "avg_mpg_min": 14.0052739234155, "avg_mpg_max": 20.55764396115954}, "sampler_perf": {"mean_env_wait_ms": 1670.1093003574792, "mean_processing_ms": 430.5544359845478, "mean_inference_ms": 1.9823995230643374}, "off_policy_estimator": {}, "info": {"num_steps_trained": 5532160, "num_steps_sampled": 85000, "sample_time_ms": 2041632.773, "load_time_ms": 443.865, "grad_time_ms": 189596.24, "update_time_ms": 113.251, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 1076.412353515625, "policy_loss": -0.0006808983744122088, "vf_loss": 1076.4130859375, "vf_explained_var": 0.00635658809915185, "vf_preds": 67.06362915039062, "kl": 0.007396731059998274, "entropy": 1.3012518882751465, "entropy_coeff": 0.0, "advantages": 5.862341822648887e-06, "rewards": 1.7216050624847412}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 85000, "episodes_total": 85, "training_iteration": 5, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_04-22-26", "timestamp": 1599020546, "time_this_iter_s": 2168.85209941864, "time_total_s": 11160.793151140213, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc75f2f0>", "on_episode_step": ".on_episode_step at 0x7f3afc75f400>", "on_episode_end": ".on_episode_end at 0x7f3afc75f378>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc75f840>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 11160.793151140213, "timesteps_since_restore": 85000, "iterations_since_restore": 5, "perf": {"cpu_util_percent": 90.44087378640776, "ram_util_percent": 29.50430420711974}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 120501.62598433408, "episode_reward_min": 49824.838241618454, "episode_reward_mean": 105406.28169113278, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 573.698932479357}, "policy_reward_mean": {"av": 370.61383808984493}, "custom_metrics": {"avg_speed_mean": 5.337685738813261, "avg_speed_min": 3.1503079652604264, "avg_speed_max": 7.0043220707592955, "avg_speed_avs_mean": 5.030142287375263, "avg_speed_avs_min": 2.2852643750408235, "avg_speed_avs_max": 6.427311805030021, "avg_headway_avs_mean": 16.685520010210347, "avg_headway_avs_min": 8.743219719761651, "avg_headway_avs_max": 31.282910447688586, "std_headway_avs_mean": 22.52731457438443, "std_headway_avs_min": 10.685772867313764, "std_headway_avs_max": 49.986159831576195, "avg_accel_avs_mean": 0.7417581455141067, "avg_accel_avs_min": 0.6452753112046652, "avg_accel_avs_max": 0.7748883491456845, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 18.199824857314294, "avg_mpg_per_veh_min": 14.022551226866414, "avg_mpg_per_veh_max": 20.779538032540017, "num_cars_mean": 825.445642107892, "num_cars_min": 707.898, "num_cars_max": 886.7102897102897, "avg_mpg_mean": 18.157163996991535, "avg_mpg_min": 14.0052739234155, "avg_mpg_max": 20.55764396115954}, "sampler_perf": {"mean_env_wait_ms": 1647.924347051285, "mean_processing_ms": 429.71684644423715, "mean_inference_ms": 1.9729568145191851}, "off_policy_estimator": {}, "info": {"num_steps_trained": 6667520, "num_steps_sampled": 102000, "sample_time_ms": 2035499.071, "load_time_ms": 434.814, "grad_time_ms": 189971.936, "update_time_ms": 96.017, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 1106.4464111328125, "policy_loss": -0.0003084279887843877, "vf_loss": 1106.44677734375, "vf_explained_var": 0.0023608864285051823, "vf_preds": 74.18597412109375, "kl": 0.006067431066185236, "entropy": 1.2881662845611572, "entropy_coeff": 0.0, "advantages": -4.655764485050895e-07, "rewards": 1.709161639213562}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 102000, "episodes_total": 102, "training_iteration": 6, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_04-59-04", "timestamp": 1599022744, "time_this_iter_s": 2197.4828565120697, "time_total_s": 13358.276007652283, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc75f158>", "on_episode_step": ".on_episode_step at 0x7f3afc75f1e0>", "on_episode_end": ".on_episode_end at 0x7f3afc75fe18>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc75f488>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 13358.276007652283, "timesteps_since_restore": 102000, "iterations_since_restore": 6, "perf": {"cpu_util_percent": 90.38335463258787, "ram_util_percent": 29.629680511182105}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 120501.62598433408, "episode_reward_min": 100594.12636840844, "episode_reward_mean": 115826.36714309959, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 573.698932479357}, "policy_reward_mean": {"av": 382.9857062563225}, "custom_metrics": {"avg_speed_mean": 5.601152824683562, "avg_speed_min": 5.350478353229421, "avg_speed_max": 6.816540682326582, "avg_speed_avs_mean": 5.45856173159692, "avg_speed_avs_min": 5.321753123591306, "avg_speed_avs_max": 6.157452866352753, "avg_headway_avs_mean": 13.07861774808128, "avg_headway_avs_min": 7.92975992366131, "avg_headway_avs_max": 30.662287536853945, "std_headway_avs_mean": 16.773763073946938, "std_headway_avs_min": 9.563802078059059, "std_headway_avs_max": 38.88536765922133, "avg_accel_avs_mean": 0.7569400662583738, "avg_accel_avs_min": 0.7336147184377249, "avg_accel_avs_max": 0.7748883491456845, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 18.67770496156361, "avg_mpg_per_veh_min": 17.53989052979285, "avg_mpg_per_veh_max": 20.275220615161334, "num_cars_mean": 833.5897899999999, "num_cars_min": 712.964, "num_cars_max": 869.259, "avg_mpg_mean": 18.63431918707348, "avg_mpg_min": 17.521554724453306, "avg_mpg_max": 20.190772197712686}, "sampler_perf": {"mean_env_wait_ms": 1589.8801925905436, "mean_processing_ms": 428.0063749450761, "mean_inference_ms": 1.9536932034635577}, "off_policy_estimator": {}, "info": {"num_steps_trained": 7813504, "num_steps_sampled": 119000, "sample_time_ms": 2033421.259, "load_time_ms": 434.841, "grad_time_ms": 189871.586, "update_time_ms": 83.454, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 1177.141845703125, "policy_loss": -0.00019616022473201156, "vf_loss": 1177.14208984375, "vf_explained_var": 0.007499944418668747, "vf_preds": 81.19461822509766, "kl": 0.006189987063407898, "entropy": 1.2731612920761108, "entropy_coeff": 0.0, "advantages": -4.385911324789049e-06, "rewards": 1.6928813457489014}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 119000, "episodes_total": 119, "training_iteration": 7, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_05-35-55", "timestamp": 1599024955, "time_this_iter_s": 2211.0357899665833, "time_total_s": 15569.311797618866, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc75fb70>", "on_episode_step": ".on_episode_step at 0x7f3afc75f7b8>", "on_episode_end": ".on_episode_end at 0x7f3afc75fae8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc75fd08>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 15569.311797618866, "timesteps_since_restore": 119000, "iterations_since_restore": 7, "perf": {"cpu_util_percent": 90.37726262305493, "ram_util_percent": 29.68034296602096}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 120501.62598433408, "episode_reward_min": 113326.62496026317, "episode_reward_mean": 118102.15166980994, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 571.1994129650835}, "policy_reward_mean": {"av": 382.5418704687266}, "custom_metrics": {"avg_speed_mean": 5.463120876683149, "avg_speed_min": 5.2985152961743385, "avg_speed_max": 5.7557334565463405, "avg_speed_avs_mean": 5.413279316278853, "avg_speed_avs_min": 5.289638923841501, "avg_speed_avs_max": 5.650648028324797, "avg_headway_avs_mean": 10.157292181841763, "avg_headway_avs_min": 7.658305618504542, "avg_headway_avs_max": 15.291810496119771, "std_headway_avs_mean": 12.851452554828064, "std_headway_avs_min": 8.950068866748156, "std_headway_avs_max": 21.06558729002542, "avg_accel_avs_mean": 0.7590513118114813, "avg_accel_avs_min": 0.7336147184377249, "avg_accel_avs_max": 0.7748883491456845, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 18.342892467854718, "avg_mpg_per_veh_min": 17.42638811985231, "avg_mpg_per_veh_max": 19.6266902288111, "num_cars_mean": 852.9516199999999, "num_cars_min": 815.57, "num_cars_max": 873.829, "avg_mpg_mean": 18.31160763644984, "avg_mpg_min": 17.411479879598613, "avg_mpg_max": 19.57497293471634}, "sampler_perf": {"mean_env_wait_ms": 1578.1932315108504, "mean_processing_ms": 427.2512192203174, "mean_inference_ms": 1.9472745385088965}, "off_policy_estimator": {}, "info": {"num_steps_trained": 8963072, "num_steps_sampled": 136000, "sample_time_ms": 2032861.067, "load_time_ms": 434.382, "grad_time_ms": 189396.348, "update_time_ms": 73.81, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 1248.03759765625, "policy_loss": -8.216674905270338e-05, "vf_loss": 1248.0377197265625, "vf_explained_var": 0.10780216753482819, "vf_preds": 88.37272644042969, "kl": 0.006167074665427208, "entropy": 1.266662836074829, "entropy_coeff": 0.0, "advantages": 5.787052941741422e-06, "rewards": 1.6886628866195679}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 136000, "episodes_total": 136, "training_iteration": 8, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_06-12-51", "timestamp": 1599027171, "time_this_iter_s": 2215.825352668762, "time_total_s": 17785.137150287628, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc75f6a8>", "on_episode_step": ".on_episode_step at 0x7f3afc75f598>", "on_episode_end": ".on_episode_end at 0x7f3afc75f510>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc75ff28>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 17785.137150287628, "timesteps_since_restore": 136000, "iterations_since_restore": 8, "perf": {"cpu_util_percent": 90.30063371356147, "ram_util_percent": 29.76587452471483}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 120501.62598433408, "episode_reward_min": 116738.71118546872, "episode_reward_mean": 118532.31588856528, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 571.1994129650835}, "policy_reward_mean": {"av": 381.3781077495667}, "custom_metrics": {"avg_speed_mean": 5.412053942505457, "avg_speed_min": 5.290275093204667, "avg_speed_max": 5.611816820983635, "avg_speed_avs_mean": 5.3827756953435335, "avg_speed_avs_min": 5.2886324580884105, "avg_speed_avs_max": 5.548404509391565, "avg_headway_avs_mean": 9.044729006460445, "avg_headway_avs_min": 7.175088997339596, "avg_headway_avs_max": 12.15852967842004, "std_headway_avs_mean": 11.218573722419386, "std_headway_avs_min": 8.238498060563565, "std_headway_avs_max": 16.356905308508455, "avg_accel_avs_mean": 0.7587232172581373, "avg_accel_avs_min": 0.7336147184377249, "avg_accel_avs_max": 0.7748883491456845, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 18.081188060155704, "avg_mpg_per_veh_min": 17.16663117074266, "avg_mpg_per_veh_max": 19.293587543321113, "num_cars_mean": 860.6662399999998, "num_cars_min": 836.771, "num_cars_max": 875.302, "avg_mpg_mean": 18.054491935261353, "avg_mpg_min": 17.159649545484612, "avg_mpg_max": 19.221167311152932}, "sampler_perf": {"mean_env_wait_ms": 1574.813803274435, "mean_processing_ms": 426.04506014565965, "mean_inference_ms": 1.9434809428137327}, "off_policy_estimator": {}, "info": {"num_steps_trained": 10116224, "num_steps_sampled": 153000, "sample_time_ms": 2032601.832, "load_time_ms": 431.028, "grad_time_ms": 189822.652, "update_time_ms": 66.566, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 1338.20361328125, "policy_loss": -5.598955613095313e-05, "vf_loss": 1338.2037353515625, "vf_explained_var": 0.16906680166721344, "vf_preds": 92.73361206054688, "kl": 0.005495058372616768, "entropy": 1.2682840824127197, "entropy_coeff": 0.0, "advantages": -5.110851816425566e-06, "rewards": 1.6845884323120117}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 153000, "episodes_total": 153, "training_iteration": 9, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_06-49-55", "timestamp": 1599029395, "time_this_iter_s": 2224.5567462444305, "time_total_s": 20009.69389653206, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc75fa60>", "on_episode_step": ".on_episode_step at 0x7f3afc75fd90>", "on_episode_end": ".on_episode_end at 0x7f3afc75fbf8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc75f268>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 20009.69389653206, "timesteps_since_restore": 153000, "iterations_since_restore": 9, "perf": {"cpu_util_percent": 90.32045454545455, "ram_util_percent": 29.79084595959596}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 120501.62598433408, "episode_reward_min": 116738.71118546872, "episode_reward_mean": 118779.47418229684, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 571.1994129650835}, "policy_reward_mean": {"av": 380.84992363183557}, "custom_metrics": {"avg_speed_mean": 5.382656069845253, "avg_speed_min": 5.278263880882649, "avg_speed_max": 5.509260175747669, "avg_speed_avs_mean": 5.363012281268494, "avg_speed_avs_min": 5.2610447978712696, "avg_speed_avs_max": 5.475550380553723, "avg_headway_avs_mean": 8.412822823751618, "avg_headway_avs_min": 7.175088997339596, "avg_headway_avs_max": 10.361490535575385, "std_headway_avs_mean": 10.21821642063553, "std_headway_avs_min": 8.238498060563565, "std_headway_avs_max": 13.503091880563334, "avg_accel_avs_mean": 0.7601100001048802, "avg_accel_avs_min": 0.7336147184377249, "avg_accel_avs_max": 0.7821300391306429, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 17.928651959344315, "avg_mpg_per_veh_min": 17.16663117074266, "avg_mpg_per_veh_max": 18.776813684757492, "num_cars_mean": 865.1029800000001, "num_cars_min": 849.155, "num_cars_max": 876.352, "avg_mpg_mean": 17.90494891005331, "avg_mpg_min": 17.159649545484612, "avg_mpg_max": 18.750120752029655}, "sampler_perf": {"mean_env_wait_ms": 1574.1214092947814, "mean_processing_ms": 424.9046363575242, "mean_inference_ms": 1.9407349038739747}, "off_policy_estimator": {}, "info": {"num_steps_trained": 11270784, "num_steps_sampled": 170000, "sample_time_ms": 2032561.668, "load_time_ms": 432.186, "grad_time_ms": 189998.513, "update_time_ms": 60.745, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 1521.97607421875, "policy_loss": 4.975534011464333e-06, "vf_loss": 1521.9759521484375, "vf_explained_var": 0.18849284946918488, "vf_preds": 98.10453033447266, "kl": 0.005310830194503069, "entropy": 1.2807040214538574, "entropy_coeff": 0.0, "advantages": 1.7932143236976117e-05, "rewards": 1.6936533451080322}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 170000, "episodes_total": 170, "training_iteration": 10, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_07-27-00", "timestamp": 1599031620, "time_this_iter_s": 2224.597925424576, "time_total_s": 22234.291821956635, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc71e488>", "on_episode_step": ".on_episode_step at 0x7f3afc71e268>", "on_episode_end": ".on_episode_end at 0x7f3b7341de18>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3b7341df28>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 22234.291821956635, "timesteps_since_restore": 170000, "iterations_since_restore": 10, "perf": {"cpu_util_percent": 90.02597033764594, "ram_util_percent": 30.023698327548125}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 121307.94983268024, "episode_reward_min": 116738.71118546872, "episode_reward_mean": 119131.3039272643, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 566.7550613795677}, "policy_reward_mean": {"av": 381.4154572813738}, "custom_metrics": {"avg_speed_mean": 5.359933273968397, "avg_speed_min": 5.278263880882649, "avg_speed_max": 5.449489852845025, "avg_speed_avs_mean": 5.344880149132677, "avg_speed_avs_min": 5.2610447978712696, "avg_speed_avs_max": 5.426572695876386, "avg_headway_avs_mean": 8.055970351469984, "avg_headway_avs_min": 7.175088997339596, "avg_headway_avs_max": 9.429925062195265, "std_headway_avs_mean": 9.625281288877797, "std_headway_avs_min": 8.238498060563565, "std_headway_avs_max": 11.99862750860526, "avg_accel_avs_mean": 0.7586019997400248, "avg_accel_avs_min": 0.7336147184377249, "avg_accel_avs_max": 0.7821300391306429, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 17.90973318167203, "avg_mpg_per_veh_min": 17.16663117074266, "avg_mpg_per_veh_max": 18.383930029471333, "num_cars_mean": 868.3603099999999, "num_cars_min": 856.101, "num_cars_max": 877.08, "avg_mpg_mean": 17.88697746422036, "avg_mpg_min": 17.159649545484612, "avg_mpg_max": 18.36035205336454}, "sampler_perf": {"mean_env_wait_ms": 1574.519504741383, "mean_processing_ms": 424.0362672878738, "mean_inference_ms": 1.9385644873036794}, "off_policy_estimator": {}, "info": {"num_steps_trained": 12428544, "num_steps_sampled": 187000, "sample_time_ms": 1987363.106, "load_time_ms": 416.133, "grad_time_ms": 186817.028, "update_time_ms": 8.139, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 1702.081787109375, "policy_loss": -2.3841066649765708e-05, "vf_loss": 1702.08203125, "vf_explained_var": 0.2250986546278, "vf_preds": 103.72588348388672, "kl": 0.006048419047147036, "entropy": 1.2268348932266235, "entropy_coeff": 0.0, "advantages": 6.791436817366048e-07, "rewards": 1.7016113996505737}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 187000, "episodes_total": 187, "training_iteration": 11, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_08-03-56", "timestamp": 1599033836, "time_this_iter_s": 2216.2808804512024, "time_total_s": 24450.572702407837, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afff9bea0>", "on_episode_step": ".on_episode_step at 0x7f3afff9bd90>", "on_episode_end": ".on_episode_end at 0x7f3afff9bd08>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afff9b378>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 24450.572702407837, "timesteps_since_restore": 187000, "iterations_since_restore": 11, "perf": {"cpu_util_percent": 90.0654309252218, "ram_util_percent": 30.11723700887199}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 121307.94983268024, "episode_reward_min": 117264.01965883735, "episode_reward_mean": 119324.0722195423, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 560.056148990761}, "policy_reward_mean": {"av": 381.531805658009}, "custom_metrics": {"avg_speed_mean": 5.349250698394234, "avg_speed_min": 5.278263880882649, "avg_speed_max": 5.425695499282145, "avg_speed_avs_mean": 5.335967967135548, "avg_speed_avs_min": 5.2610447978712696, "avg_speed_avs_max": 5.411264231394141, "avg_headway_avs_mean": 7.858340114713448, "avg_headway_avs_min": 7.175088997339596, "avg_headway_avs_max": 8.463396619045929, "std_headway_avs_mean": 9.259175201864226, "std_headway_avs_min": 8.238498060563565, "std_headway_avs_max": 10.455265028995186, "avg_accel_avs_mean": 0.756902973843408, "avg_accel_avs_min": 0.7335509944637656, "avg_accel_avs_max": 0.7821300391306429, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 17.891314095798894, "avg_mpg_per_veh_min": 17.16663117074266, "avg_mpg_per_veh_max": 18.3722128455099, "num_cars_mean": 870.04872, "num_cars_min": 861.967, "num_cars_max": 877.08, "avg_mpg_mean": 17.8693205098089, "avg_mpg_min": 17.159649545484612, "avg_mpg_max": 18.332222632508334}, "sampler_perf": {"mean_env_wait_ms": 1575.2759407118904, "mean_processing_ms": 423.2855589786875, "mean_inference_ms": 1.937167648828551}, "off_policy_estimator": {}, "info": {"num_steps_trained": 13583872, "num_steps_sampled": 204000, "sample_time_ms": 2009600.96, "load_time_ms": 425.73, "grad_time_ms": 189506.906, "update_time_ms": 7.979, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 1909.294921875, "policy_loss": 0.0001959388901013881, "vf_loss": 1909.294921875, "vf_explained_var": 0.2512345314025879, "vf_preds": 108.81951904296875, "kl": 0.005425139330327511, "entropy": 1.1910916566848755, "entropy_coeff": 0.0, "advantages": 4.0973304749059025e-06, "rewards": 1.6947407722473145}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 204000, "episodes_total": 204, "training_iteration": 12, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_08-41-01", "timestamp": 1599036061, "time_this_iter_s": 2224.221389770508, "time_total_s": 26674.794092178345, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc704840>", "on_episode_step": ".on_episode_step at 0x7f3afc7048c8>", "on_episode_end": ".on_episode_end at 0x7f3afc704950>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc7049d8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 26674.794092178345, "timesteps_since_restore": 204000, "iterations_since_restore": 12, "perf": {"cpu_util_percent": 90.00684974747475, "ram_util_percent": 29.934690656565653}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 121563.1337298867, "episode_reward_min": 117264.01965883735, "episode_reward_mean": 119623.00210374853, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 560.056148990761}, "policy_reward_mean": {"av": 382.51207784270287}, "custom_metrics": {"avg_speed_mean": 5.346084757135717, "avg_speed_min": 5.278263880882649, "avg_speed_max": 5.421651578491962, "avg_speed_avs_mean": 5.332582317993154, "avg_speed_avs_min": 5.2610447978712696, "avg_speed_avs_max": 5.409551125290047, "avg_headway_avs_mean": 7.92235384215814, "avg_headway_avs_min": 7.175088997339596, "avg_headway_avs_max": 8.714032117444868, "std_headway_avs_mean": 9.200338730652263, "std_headway_avs_min": 8.238498060563565, "std_headway_avs_max": 10.0189561310151, "avg_accel_avs_mean": 0.7550674949154036, "avg_accel_avs_min": 0.7312783720707061, "avg_accel_avs_max": 0.7821300391306429, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 17.96402951842984, "avg_mpg_per_veh_min": 17.16663117074266, "avg_mpg_per_veh_max": 18.484327974191977, "num_cars_mean": 870.4815900000001, "num_cars_min": 861.967, "num_cars_max": 877.08, "avg_mpg_mean": 17.941691259074965, "avg_mpg_min": 17.159649545484612, "avg_mpg_max": 18.46052500209466}, "sampler_perf": {"mean_env_wait_ms": 1576.0312898886345, "mean_processing_ms": 422.64250791637204, "mean_inference_ms": 1.936006561504521}, "off_policy_estimator": {}, "info": {"num_steps_trained": 14734464, "num_steps_sampled": 221000, "sample_time_ms": 2017080.721, "load_time_ms": 431.985, "grad_time_ms": 189834.041, "update_time_ms": 8.048, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 2219.32958984375, "policy_loss": 0.00014577151159755886, "vf_loss": 2219.329345703125, "vf_explained_var": 0.2551387548446655, "vf_preds": 113.63782501220703, "kl": 0.004434822592884302, "entropy": 1.1568522453308105, "entropy_coeff": 0.0, "advantages": 1.7590034531167476e-06, "rewards": 1.7125211954116821}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 221000, "episodes_total": 221, "training_iteration": 13, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_09-17-57", "timestamp": 1599038277, "time_this_iter_s": 2216.3313598632812, "time_total_s": 28891.125452041626, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afff9bb70>", "on_episode_step": ".on_episode_step at 0x7f3afff9bbf8>", "on_episode_end": ".on_episode_end at 0x7f3afff9b8c8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afff9b9d8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 28891.125452041626, "timesteps_since_restore": 221000, "iterations_since_restore": 13, "perf": {"cpu_util_percent": 90.40728539752932, "ram_util_percent": 29.932752613240417}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 121563.1337298867, "episode_reward_min": 117304.69246500851, "episode_reward_mean": 119891.00276368452, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 574.8529083139975}, "policy_reward_mean": {"av": 383.49167630644683}, "custom_metrics": {"avg_speed_mean": 5.345249294014289, "avg_speed_min": 5.276867536832988, "avg_speed_max": 5.421651578491962, "avg_speed_avs_mean": 5.331602732678717, "avg_speed_avs_min": 5.2610447978712696, "avg_speed_avs_max": 5.409551125290047, "avg_headway_avs_mean": 8.031380788274951, "avg_headway_avs_min": 7.175088997339596, "avg_headway_avs_max": 8.919925267047375, "std_headway_avs_mean": 9.224092064379304, "std_headway_avs_min": 8.238498060563565, "std_headway_avs_max": 10.057762195602479, "avg_accel_avs_mean": 0.7539997514530019, "avg_accel_avs_min": 0.7312783720707061, "avg_accel_avs_max": 0.7821300391306429, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 18.042725217121347, "avg_mpg_per_veh_min": 17.16663117074266, "avg_mpg_per_veh_max": 18.648120622563752, "num_cars_mean": 870.5064700000001, "num_cars_min": 863.842, "num_cars_max": 877.08, "avg_mpg_mean": 18.01974479123578, "avg_mpg_min": 17.159649545484612, "avg_mpg_max": 18.597933064907927}, "sampler_perf": {"mean_env_wait_ms": 1576.6590768050391, "mean_processing_ms": 422.10930799739015, "mean_inference_ms": 1.9350315664609337}, "off_policy_estimator": {}, "info": {"num_steps_trained": 15885568, "num_steps_sampled": 238000, "sample_time_ms": 2020288.104, "load_time_ms": 433.954, "grad_time_ms": 189758.131, "update_time_ms": 7.932, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 2498.629150390625, "policy_loss": -0.00012051299563609064, "vf_loss": 2498.62939453125, "vf_explained_var": 0.25726938247680664, "vf_preds": 117.88414764404297, "kl": 0.005063860677182674, "entropy": 1.1688635349273682, "entropy_coeff": 0.0, "advantages": 1.4100942280492745e-06, "rewards": 1.7089608907699585}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 238000, "episodes_total": 238, "training_iteration": 14, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_09-54-47", "timestamp": 1599040487, "time_this_iter_s": 2209.5433192253113, "time_total_s": 31100.668771266937, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc75f8c8>", "on_episode_step": ".on_episode_step at 0x7f3afc75fd90>", "on_episode_end": ".on_episode_end at 0x7f3afc71e950>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc71e6a8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 31100.668771266937, "timesteps_since_restore": 238000, "iterations_since_restore": 14, "perf": {"cpu_util_percent": 90.32551636479185, "ram_util_percent": 29.93844931680966}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 123118.70260606187, "episode_reward_min": 118250.25453960584, "episode_reward_mean": 120308.15042356087, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 574.8529083139975}, "policy_reward_mean": {"av": 385.3930564229773}, "custom_metrics": {"avg_speed_mean": 5.34393585427646, "avg_speed_min": 5.276867536832988, "avg_speed_max": 5.394941530856778, "avg_speed_avs_mean": 5.328444457814203, "avg_speed_avs_min": 5.2610447978712696, "avg_speed_avs_max": 5.389170889477089, "avg_headway_avs_mean": 8.343464485011303, "avg_headway_avs_min": 7.459939080193464, "avg_headway_avs_max": 9.918288673934098, "std_headway_avs_mean": 9.385055442358867, "std_headway_avs_min": 8.419413399413449, "std_headway_avs_max": 10.635616400096074, "avg_accel_avs_mean": 0.7529865608989144, "avg_accel_avs_min": 0.7312783720707061, "avg_accel_avs_max": 0.7821300391306429, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 18.237832737942334, "avg_mpg_per_veh_min": 17.514745557959877, "avg_mpg_per_veh_max": 19.27930456916758, "num_cars_mean": 869.9381400000001, "num_cars_min": 862.205, "num_cars_max": 877.08, "avg_mpg_mean": 18.212215654558396, "avg_mpg_min": 17.496137329696833, "avg_mpg_max": 19.24198778595331}, "sampler_perf": {"mean_env_wait_ms": 1577.0998879146712, "mean_processing_ms": 421.64700282695935, "mean_inference_ms": 1.9340731632738497}, "off_policy_estimator": {}, "info": {"num_steps_trained": 17032704, "num_steps_sampled": 255000, "sample_time_ms": 2023863.111, "load_time_ms": 436.265, "grad_time_ms": 190721.966, "update_time_ms": 7.759, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 2857.260498046875, "policy_loss": 0.00011845603148685768, "vf_loss": 2857.260498046875, "vf_explained_var": 0.24712079763412476, "vf_preds": 119.84867858886719, "kl": 0.004157697781920433, "entropy": 1.1496570110321045, "entropy_coeff": 0.0, "advantages": -1.087875375560543e-06, "rewards": 1.7325016260147095}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 255000, "episodes_total": 255, "training_iteration": 15, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_10-31-41", "timestamp": 1599042701, "time_this_iter_s": 2214.2694425582886, "time_total_s": 33314.938213825226, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc71e840>", "on_episode_step": ".on_episode_step at 0x7f3afc71e048>", "on_episode_end": ".on_episode_end at 0x7f3afc71e598>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc71e7b8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 33314.938213825226, "timesteps_since_restore": 255000, "iterations_since_restore": 15, "perf": {"cpu_util_percent": 90.4453536314621, "ram_util_percent": 29.940437678401526}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 123118.70260606187, "episode_reward_min": 118631.84594306789, "episode_reward_mean": 120550.85454693026, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 577.8644746799712}, "policy_reward_mean": {"av": 386.77763907510985}, "custom_metrics": {"avg_speed_mean": 5.345161821525397, "avg_speed_min": 5.276867536832988, "avg_speed_max": 5.403372302846698, "avg_speed_avs_mean": 5.32736317762261, "avg_speed_avs_min": 5.265286510305824, "avg_speed_avs_max": 5.3833177883894825, "avg_headway_avs_mean": 8.714031568505039, "avg_headway_avs_min": 7.459939080193464, "avg_headway_avs_max": 10.342775798017534, "std_headway_avs_mean": 9.618629172947978, "std_headway_avs_min": 8.419413399413449, "std_headway_avs_max": 10.883261215886664, "avg_accel_avs_mean": 0.7519140528501825, "avg_accel_avs_min": 0.7312783720707061, "avg_accel_avs_max": 0.7822048362540943, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 18.383965447469652, "avg_mpg_per_veh_min": 17.60401050394941, "avg_mpg_per_veh_max": 19.27930456916758, "num_cars_mean": 868.8772700000002, "num_cars_min": 861.488, "num_cars_max": 877.08, "avg_mpg_mean": 18.356554636831113, "avg_mpg_min": 17.59999340578443, "avg_mpg_max": 19.24198778595331}, "sampler_perf": {"mean_env_wait_ms": 1577.3357944039424, "mean_processing_ms": 421.1650412584571, "mean_inference_ms": 1.9333174326779894}, "off_policy_estimator": {}, "info": {"num_steps_trained": 18175744, "num_steps_sampled": 272000, "sample_time_ms": 2023502.741, "load_time_ms": 439.973, "grad_time_ms": 190075.438, "update_time_ms": 7.475, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3188.12353515625, "policy_loss": 1.2626305760932155e-06, "vf_loss": 3188.123291015625, "vf_explained_var": 0.23051680624485016, "vf_preds": 122.627685546875, "kl": 0.004072967451065779, "entropy": 1.143757700920105, "entropy_coeff": 0.0, "advantages": -1.1981988791376352e-05, "rewards": 1.7337217330932617}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 272000, "episodes_total": 272, "training_iteration": 16, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_11-08-08", "timestamp": 1599044888, "time_this_iter_s": 2187.4395565986633, "time_total_s": 35502.37777042389, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3b003b3f28>", "on_episode_step": ".on_episode_step at 0x7f3b003b37b8>", "on_episode_end": ".on_episode_end at 0x7f3b003b3c80>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3b003b3ea0>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 35502.37777042389, "timesteps_since_restore": 272000, "iterations_since_restore": 16, "perf": {"cpu_util_percent": 90.20994865211809, "ram_util_percent": 29.952053915275997}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 123118.70260606187, "episode_reward_min": 118631.84594306789, "episode_reward_mean": 120507.46152748268, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 577.8644746799712}, "policy_reward_mean": {"av": 387.1974473138279}, "custom_metrics": {"avg_speed_mean": 5.351480491867342, "avg_speed_min": 5.276867536832988, "avg_speed_max": 5.4148556865601805, "avg_speed_avs_mean": 5.332654080607074, "avg_speed_avs_min": 5.265286510305824, "avg_speed_avs_max": 5.399463845248409, "avg_headway_avs_mean": 9.10446308644008, "avg_headway_avs_min": 7.655483893314458, "avg_headway_avs_max": 10.441814669117026, "std_headway_avs_mean": 9.836086923075484, "std_headway_avs_min": 8.687177577201222, "std_headway_avs_max": 10.883261215886664, "avg_accel_avs_mean": 0.7541747862062319, "avg_accel_avs_min": 0.7312783720707061, "avg_accel_avs_max": 0.7822048362540943, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 18.43152611581485, "avg_mpg_per_veh_min": 17.60401050394941, "avg_mpg_per_veh_max": 19.27930456916758, "num_cars_mean": 867.09636, "num_cars_min": 856.088, "num_cars_max": 875.65, "avg_mpg_mean": 18.403488253434794, "avg_mpg_min": 17.59999340578443, "avg_mpg_max": 19.24198778595331}, "sampler_perf": {"mean_env_wait_ms": 1577.3145973263888, "mean_processing_ms": 420.61475289912744, "mean_inference_ms": 1.9326269487340513}, "off_policy_estimator": {}, "info": {"num_steps_trained": 19313152, "num_steps_sampled": 289000, "sample_time_ms": 2020589.598, "load_time_ms": 437.628, "grad_time_ms": 189106.043, "update_time_ms": 7.47, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3427.173828125, "policy_loss": 7.21618634997867e-05, "vf_loss": 3427.173828125, "vf_explained_var": 0.2360333502292633, "vf_preds": 124.64932250976562, "kl": 0.003702861722558737, "entropy": 1.0856871604919434, "entropy_coeff": 0.0, "advantages": 1.2924696420668624e-05, "rewards": 1.7261053323745728}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 289000, "episodes_total": 289, "training_iteration": 17, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_11-44-21", "timestamp": 1599047061, "time_this_iter_s": 2172.1764299869537, "time_total_s": 37674.55420041084, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc75fe18>", "on_episode_step": ".on_episode_step at 0x7f3afc75f158>", "on_episode_end": ".on_episode_end at 0x7f3afc75f048>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc75f488>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 37674.55420041084, "timesteps_since_restore": 289000, "iterations_since_restore": 17, "perf": {"cpu_util_percent": 90.54683257918553, "ram_util_percent": 30.026599870717515}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 123118.70260606187, "episode_reward_min": 118534.3730828062, "episode_reward_mean": 120507.32020792307, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 577.8644746799712}, "policy_reward_mean": {"av": 387.607977510206}, "custom_metrics": {"avg_speed_mean": 5.3506847419433665, "avg_speed_min": 5.276867536832988, "avg_speed_max": 5.4148556865601805, "avg_speed_avs_mean": 5.331198195653265, "avg_speed_avs_min": 5.265286510305824, "avg_speed_avs_max": 5.399463845248409, "avg_headway_avs_mean": 9.37019204808072, "avg_headway_avs_min": 8.199462891141241, "avg_headway_avs_max": 10.441814669117026, "std_headway_avs_mean": 9.954878342223612, "std_headway_avs_min": 9.061658486984664, "std_headway_avs_max": 10.883261215886664, "avg_accel_avs_mean": 0.7587295004432463, "avg_accel_avs_min": 0.7312783720707061, "avg_accel_avs_max": 0.784211744381461, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 18.470940614329578, "avg_mpg_per_veh_min": 17.740744323786593, "avg_mpg_per_veh_max": 19.27930456916758, "num_cars_mean": 866.25422, "num_cars_min": 856.088, "num_cars_max": 872.333, "avg_mpg_mean": 18.442540852392327, "avg_mpg_min": 17.7283512555159, "avg_mpg_max": 19.24198778595331}, "sampler_perf": {"mean_env_wait_ms": 1577.1772586099146, "mean_processing_ms": 419.7944154408089, "mean_inference_ms": 1.9318474364808653}, "off_policy_estimator": {}, "info": {"num_steps_trained": 20457856, "num_steps_sampled": 306000, "sample_time_ms": 2014975.165, "load_time_ms": 435.769, "grad_time_ms": 188228.788, "update_time_ms": 7.49, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3627.134521484375, "policy_loss": 0.00013671170745510608, "vf_loss": 3627.134033203125, "vf_explained_var": 0.24064086377620697, "vf_preds": 123.45279693603516, "kl": 0.003551310859620571, "entropy": 1.1051963567733765, "entropy_coeff": 0.0, "advantages": -5.382640210882528e-06, "rewards": 1.7106589078903198}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 306000, "episodes_total": 306, "training_iteration": 18, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_12-20-12", "timestamp": 1599049212, "time_this_iter_s": 2150.9124476909637, "time_total_s": 39825.46664810181, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc75f840>", "on_episode_step": ".on_episode_step at 0x7f3afc75f9d8>", "on_episode_end": ".on_episode_end at 0x7f3afc75f620>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc75fc80>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 39825.46664810181, "timesteps_since_restore": 306000, "iterations_since_restore": 18, "perf": {"cpu_util_percent": 90.666318537859, "ram_util_percent": 30.054830287206265}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 123118.70260606187, "episode_reward_min": 118534.3730828062, "episode_reward_mean": 120598.6824105813, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 577.8644746799712}, "policy_reward_mean": {"av": 388.40155365726656}, "custom_metrics": {"avg_speed_mean": 5.336234223779533, "avg_speed_min": 5.211167920959937, "avg_speed_max": 5.4148556865601805, "avg_speed_avs_mean": 5.316527630919953, "avg_speed_avs_min": 5.188852657286994, "avg_speed_avs_max": 5.399463845248409, "avg_headway_avs_mean": 9.559000841765938, "avg_headway_avs_min": 8.242775409299451, "avg_headway_avs_max": 10.441814669117026, "std_headway_avs_mean": 9.967178116520907, "std_headway_avs_min": 9.061658486984664, "std_headway_avs_max": 10.883261215886664, "avg_accel_avs_mean": 0.7628382675147586, "avg_accel_avs_min": 0.7367351634241566, "avg_accel_avs_max": 0.784211744381461, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 18.539676874162854, "avg_mpg_per_veh_min": 17.740744323786593, "avg_mpg_per_veh_max": 19.27930456916758, "num_cars_mean": 866.90127, "num_cars_min": 856.088, "num_cars_max": 876.498, "avg_mpg_mean": 18.510915541546908, "avg_mpg_min": 17.7283512555159, "avg_mpg_max": 19.24198778595331}, "sampler_perf": {"mean_env_wait_ms": 1577.0905682599891, "mean_processing_ms": 418.64171628430165, "mean_inference_ms": 1.9311401552975116}, "off_policy_estimator": {}, "info": {"num_steps_trained": 21612672, "num_steps_sampled": 323000, "sample_time_ms": 2010729.464, "load_time_ms": 440.481, "grad_time_ms": 187248.86, "update_time_ms": 7.442, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3728.15478515625, "policy_loss": 9.749025775818154e-05, "vf_loss": 3728.154296875, "vf_explained_var": 0.2171565443277359, "vf_preds": 121.5765380859375, "kl": 0.0036183164920657873, "entropy": 1.1159048080444336, "entropy_coeff": 0.0, "advantages": 3.991243374912301e-06, "rewards": 1.7142400741577148}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 323000, "episodes_total": 323, "training_iteration": 19, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_12-56-24", "timestamp": 1599051384, "time_this_iter_s": 2172.320654630661, "time_total_s": 41997.78730273247, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afff9b510>", "on_episode_step": ".on_episode_step at 0x7f3afff9b488>", "on_episode_end": ".on_episode_end at 0x7f3afff9bd90>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc75f400>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 41997.78730273247, "timesteps_since_restore": 323000, "iterations_since_restore": 19, "perf": {"cpu_util_percent": 90.5148028442146, "ram_util_percent": 30.066936005171296}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 123118.70260606187, "episode_reward_min": 118534.3730828062, "episode_reward_mean": 120715.56202626727, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 577.8644746799712}, "policy_reward_mean": {"av": 389.20415922835724}, "custom_metrics": {"avg_speed_mean": 5.317005549079885, "avg_speed_min": 5.172430183664464, "avg_speed_max": 5.4148556865601805, "avg_speed_avs_mean": 5.297003566238821, "avg_speed_avs_min": 5.15542676087789, "avg_speed_avs_max": 5.399463845248409, "avg_headway_avs_mean": 9.67558990747858, "avg_headway_avs_min": 9.130161808783265, "avg_headway_avs_max": 10.441814669117026, "std_headway_avs_mean": 9.888321742330673, "std_headway_avs_min": 8.8949135162701, "std_headway_avs_max": 10.883261215886664, "avg_accel_avs_mean": 0.7665609136996472, "avg_accel_avs_min": 0.7439313768229578, "avg_accel_avs_max": 0.784211744381461, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 18.60425160177536, "avg_mpg_per_veh_min": 17.740744323786593, "avg_mpg_per_veh_max": 19.27930456916758, "num_cars_mean": 868.4482499999998, "num_cars_min": 856.088, "num_cars_max": 883.314, "avg_mpg_mean": 18.575527059603537, "avg_mpg_min": 17.7283512555159, "avg_mpg_max": 19.24198778595331}, "sampler_perf": {"mean_env_wait_ms": 1577.0972295527774, "mean_processing_ms": 417.27335497600194, "mean_inference_ms": 1.930499667321117}, "off_policy_estimator": {}, "info": {"num_steps_trained": 22776192, "num_steps_sampled": 340000, "sample_time_ms": 2007186.454, "load_time_ms": 441.391, "grad_time_ms": 186474.034, "update_time_ms": 7.368, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3702.02587890625, "policy_loss": 0.00016934207815211266, "vf_loss": 3702.025634765625, "vf_explained_var": 0.21416132152080536, "vf_preds": 117.59202575683594, "kl": 0.0030935818795114756, "entropy": 1.1353908777236938, "entropy_coeff": 0.0, "advantages": 8.884190719982143e-06, "rewards": 1.704363465309143}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 340000, "episodes_total": 340, "training_iteration": 20, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_13-32-46", "timestamp": 1599053566, "time_this_iter_s": 2181.4547097682953, "time_total_s": 44179.24201250076, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc75f730>", "on_episode_step": ".on_episode_step at 0x7f3afc75fbf8>", "on_episode_end": ".on_episode_end at 0x7f3afc75f598>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc75f488>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 44179.24201250076, "timesteps_since_restore": 340000, "iterations_since_restore": 20, "perf": {"cpu_util_percent": 90.75271966527197, "ram_util_percent": 30.093047956227874}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 122652.02996610309, "episode_reward_min": 118534.3730828062, "episode_reward_mean": 120706.62226749385, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 573.9038677956526}, "policy_reward_mean": {"av": 389.3887617906831}, "custom_metrics": {"avg_speed_mean": 5.298971272108009, "avg_speed_min": 5.172430183664464, "avg_speed_max": 5.4148556865601805, "avg_speed_avs_mean": 5.279643041125489, "avg_speed_avs_min": 5.15542676087789, "avg_speed_avs_max": 5.399463845248409, "avg_headway_avs_mean": 9.660010725071965, "avg_headway_avs_min": 9.130161808783265, "avg_headway_avs_max": 10.441814669117026, "std_headway_avs_mean": 9.782407383107346, "std_headway_avs_min": 8.8949135162701, "std_headway_avs_max": 10.883261215886664, "avg_accel_avs_mean": 0.7693898586297861, "avg_accel_avs_min": 0.7508346435325473, "avg_accel_avs_max": 0.784211744381461, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 18.58675258653813, "avg_mpg_per_veh_min": 17.740744323786593, "avg_mpg_per_veh_max": 19.06925680663195, "num_cars_mean": 870.16847, "num_cars_min": 856.088, "num_cars_max": 883.314, "avg_mpg_mean": 18.559973898579216, "avg_mpg_min": 17.7283512555159, "avg_mpg_max": 19.038847044777384}, "sampler_perf": {"mean_env_wait_ms": 1576.9062864057714, "mean_processing_ms": 416.06100391194065, "mean_inference_ms": 1.9296488594616081}, "off_policy_estimator": {}, "info": {"num_steps_trained": 23938176, "num_steps_sampled": 357000, "sample_time_ms": 2005147.695, "load_time_ms": 429.205, "grad_time_ms": 186117.406, "update_time_ms": 7.477, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3639.150146484375, "policy_loss": 0.00018596182053443044, "vf_loss": 3639.150146484375, "vf_explained_var": 0.2030840963125229, "vf_preds": 118.67961120605469, "kl": 0.00353403203189373, "entropy": 1.1495490074157715, "entropy_coeff": 0.0, "advantages": -4.287003775971243e-06, "rewards": 1.7087461948394775}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 357000, "episodes_total": 357, "training_iteration": 21, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_14-09-18", "timestamp": 1599055758, "time_this_iter_s": 2192.1308212280273, "time_total_s": 46371.37283372879, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc704598>", "on_episode_step": ".on_episode_step at 0x7f3afc704510>", "on_episode_end": ".on_episode_end at 0x7f3afc704488>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc704378>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 46371.37283372879, "timesteps_since_restore": 357000, "iterations_since_restore": 21, "perf": {"cpu_util_percent": 90.77908392056374, "ram_util_percent": 30.128219090326713}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 122652.02996610309, "episode_reward_min": 118534.3730828062, "episode_reward_mean": 120788.46270293788, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 573.9038677956526}, "policy_reward_mean": {"av": 389.4015368094971}, "custom_metrics": {"avg_speed_mean": 5.283497587302578, "avg_speed_min": 5.172430183664464, "avg_speed_max": 5.4148556865601805, "avg_speed_avs_mean": 5.265577717320683, "avg_speed_avs_min": 5.15542676087789, "avg_speed_avs_max": 5.399463845248409, "avg_headway_avs_mean": 9.45465280664339, "avg_headway_avs_min": 8.668828471883412, "avg_headway_avs_max": 10.441814669117026, "std_headway_avs_mean": 9.51263652855188, "std_headway_avs_min": 8.57784485607507, "std_headway_avs_max": 10.77730558630447, "avg_accel_avs_mean": 0.770679600015959, "avg_accel_avs_min": 0.7550833954427503, "avg_accel_avs_max": 0.784211744381461, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 18.551328960113185, "avg_mpg_per_veh_min": 17.740744323786593, "avg_mpg_per_veh_max": 19.06925680663195, "num_cars_mean": 872.30612, "num_cars_min": 856.088, "num_cars_max": 883.314, "avg_mpg_mean": 18.525913435096626, "avg_mpg_min": 17.7283512555159, "avg_mpg_max": 19.038847044777384}, "sampler_perf": {"mean_env_wait_ms": 1576.3833431697553, "mean_processing_ms": 415.31949591446806, "mean_inference_ms": 1.9281932441590777}, "off_policy_estimator": {}, "info": {"num_steps_trained": 25102336, "num_steps_sampled": 374000, "sample_time_ms": 2002510.713, "load_time_ms": 427.809, "grad_time_ms": 185004.66, "update_time_ms": 7.667, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3509.525634765625, "policy_loss": 4.0220886148745194e-05, "vf_loss": 3509.525634765625, "vf_explained_var": 0.22132544219493866, "vf_preds": 119.16929626464844, "kl": 0.003636106150224805, "entropy": 1.1175874471664429, "entropy_coeff": 0.0, "advantages": -3.1389186005981173e-06, "rewards": 1.7067984342575073}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 374000, "episodes_total": 374, "training_iteration": 22, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_14-45-45", "timestamp": 1599057945, "time_this_iter_s": 2186.735711812973, "time_total_s": 48558.10854554176, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3b003b39d8>", "on_episode_step": ".on_episode_step at 0x7f3b003b3730>", "on_episode_end": ".on_episode_end at 0x7f3b73421048>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3b734211e0>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 48558.10854554176, "timesteps_since_restore": 374000, "iterations_since_restore": 22, "perf": {"cpu_util_percent": 90.93821451509312, "ram_util_percent": 30.067694283879256}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 122652.02996610309, "episode_reward_min": 118588.239472862, "episode_reward_mean": 121062.65976028169, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 573.9038677956526}, "policy_reward_mean": {"av": 389.8832880109551}, "custom_metrics": {"avg_speed_mean": 5.264050753133201, "avg_speed_min": 5.172430183664464, "avg_speed_max": 5.384508982414415, "avg_speed_avs_mean": 5.246892925691162, "avg_speed_avs_min": 5.15542676087789, "avg_speed_avs_max": 5.36096614073143, "avg_headway_avs_mean": 9.235460259805008, "avg_headway_avs_min": 8.62687308068883, "avg_headway_avs_max": 9.834966205651638, "std_headway_avs_mean": 9.267187942502916, "std_headway_avs_min": 8.544744912413307, "std_headway_avs_max": 10.220039900890555, "avg_accel_avs_mean": 0.7694912653985757, "avg_accel_avs_min": 0.75084693279779, "avg_accel_avs_max": 0.784211744381461, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 18.586969387786883, "avg_mpg_per_veh_min": 17.740744323786593, "avg_mpg_per_veh_max": 19.06925680663195, "num_cars_mean": 875.00508, "num_cars_min": 862.355, "num_cars_max": 883.314, "avg_mpg_mean": 18.562310588562333, "avg_mpg_min": 17.7283512555159, "avg_mpg_max": 19.038847044777384}, "sampler_perf": {"mean_env_wait_ms": 1575.6307049348281, "mean_processing_ms": 415.055530357477, "mean_inference_ms": 1.9262112898274575}, "off_policy_estimator": {}, "info": {"num_steps_trained": 26268160, "num_steps_sampled": 391000, "sample_time_ms": 2000796.86, "load_time_ms": 429.065, "grad_time_ms": 183990.028, "update_time_ms": 7.713, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3429.814208984375, "policy_loss": 0.00013203127309679985, "vf_loss": 3429.814208984375, "vf_explained_var": 0.22472494840621948, "vf_preds": 119.73706817626953, "kl": 0.0035291407257318497, "entropy": 1.0945600271224976, "entropy_coeff": 0.0, "advantages": -4.296628958400106e-06, "rewards": 1.7044821977615356}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 391000, "episodes_total": 391, "training_iteration": 23, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_15-22-14", "timestamp": 1599060134, "time_this_iter_s": 2189.046667098999, "time_total_s": 50747.15521264076, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc656158>", "on_episode_step": ".on_episode_step at 0x7f3afc6561e0>", "on_episode_end": ".on_episode_end at 0x7f3afc656268>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc6562f0>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 50747.15521264076, "timesteps_since_restore": 391000, "iterations_since_restore": 23, "perf": {"cpu_util_percent": 91.00060936497756, "ram_util_percent": 30.122065426555487}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 122652.02996610309, "episode_reward_min": 119590.04412822139, "episode_reward_mean": 121226.37616041518, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 570.9509082017903}, "policy_reward_mean": {"av": 390.1215683864814}, "custom_metrics": {"avg_speed_mean": 5.248982468999668, "avg_speed_min": 5.172430183664464, "avg_speed_max": 5.3225131372713665, "avg_speed_avs_mean": 5.232903213702072, "avg_speed_avs_min": 5.15542676087789, "avg_speed_avs_max": 5.307571893949852, "avg_headway_avs_mean": 9.01303944684724, "avg_headway_avs_min": 7.824577123701663, "avg_headway_avs_max": 9.834966205651638, "std_headway_avs_mean": 9.025229277808762, "std_headway_avs_min": 7.838192707450124, "std_headway_avs_max": 10.049790791706949, "avg_accel_avs_mean": 0.7667308816833056, "avg_accel_avs_min": 0.7496924548095301, "avg_accel_avs_max": 0.7816263239252067, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 18.57717572653809, "avg_mpg_per_veh_min": 17.80992982922428, "avg_mpg_per_veh_max": 19.06925680663195, "num_cars_mean": 877.5217400000001, "num_cars_min": 869.631, "num_cars_max": 887.736, "avg_mpg_mean": 18.553836323172543, "avg_mpg_min": 17.799061635681692, "avg_mpg_max": 19.038847044777384}, "sampler_perf": {"mean_env_wait_ms": 1574.664068367115, "mean_processing_ms": 415.35737363472043, "mean_inference_ms": 1.923894249134533}, "off_policy_estimator": {}, "info": {"num_steps_trained": 27438336, "num_steps_sampled": 408000, "sample_time_ms": 1999938.709, "load_time_ms": 434.381, "grad_time_ms": 183664.624, "update_time_ms": 7.767, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3277.0966796875, "policy_loss": 0.00011503344285301864, "vf_loss": 3277.09619140625, "vf_explained_var": 0.24595408141613007, "vf_preds": 120.6336669921875, "kl": 0.003568536601960659, "entropy": 1.085325002670288, "entropy_coeff": 0.0, "advantages": 1.667981814534869e-05, "rewards": 1.6922343969345093}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 408000, "episodes_total": 408, "training_iteration": 24, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_15-58-51", "timestamp": 1599062331, "time_this_iter_s": 2197.746801137924, "time_total_s": 52944.90201377869, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc656598>", "on_episode_step": ".on_episode_step at 0x7f3afc656620>", "on_episode_end": ".on_episode_end at 0x7f3afc6566a8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc656730>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 52944.90201377869, "timesteps_since_restore": 408000, "iterations_since_restore": 24, "perf": {"cpu_util_percent": 91.10878313637815, "ram_util_percent": 30.069115298626638}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 122731.06821363987, "episode_reward_min": 119590.04412822139, "episode_reward_mean": 121321.14345800503, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 570.9509082017903}, "policy_reward_mean": {"av": 389.74924009896245}, "custom_metrics": {"avg_speed_mean": 5.246750575044074, "avg_speed_min": 5.172430183664464, "avg_speed_max": 5.3225131372713665, "avg_speed_avs_mean": 5.23235710624581, "avg_speed_avs_min": 5.15542676087789, "avg_speed_avs_max": 5.307571893949852, "avg_headway_avs_mean": 8.717204234554465, "avg_headway_avs_min": 7.509814337476487, "avg_headway_avs_max": 9.639714965581417, "std_headway_avs_mean": 8.763568351568221, "std_headway_avs_min": 7.541473505974102, "std_headway_avs_max": 9.731351155985735, "avg_accel_avs_mean": 0.7606826729285561, "avg_accel_avs_min": 0.7227884064002857, "avg_accel_avs_max": 0.7816263239252067, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 18.517889552324696, "avg_mpg_per_veh_min": 17.80992982922428, "avg_mpg_per_veh_max": 19.06925680663195, "num_cars_mean": 878.9822, "num_cars_min": 872.349, "num_cars_max": 888.053, "avg_mpg_mean": 18.495948987703105, "avg_mpg_min": 17.799061635681692, "avg_mpg_max": 19.038847044777384}, "sampler_perf": {"mean_env_wait_ms": 1573.3744576706326, "mean_processing_ms": 416.2133379253236, "mean_inference_ms": 1.9211978891485852}, "off_policy_estimator": {}, "info": {"num_steps_trained": 28611328, "num_steps_sampled": 425000, "sample_time_ms": 1999200.359, "load_time_ms": 437.266, "grad_time_ms": 182068.084, "update_time_ms": 8.059, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3164.311767578125, "policy_loss": 0.00014171084330882877, "vf_loss": 3164.311767578125, "vf_explained_var": 0.267045259475708, "vf_preds": 120.36808776855469, "kl": 0.0038990755565464497, "entropy": 1.0270854234695435, "entropy_coeff": 0.0, "advantages": 3.5014802506339038e-06, "rewards": 1.6937894821166992}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 425000, "episodes_total": 425, "training_iteration": 25, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_16-35-23", "timestamp": 1599064523, "time_this_iter_s": 2190.989193916321, "time_total_s": 55135.89120769501, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc704e18>", "on_episode_step": ".on_episode_step at 0x7f3afc704ea0>", "on_episode_end": ".on_episode_end at 0x7f3afc704f28>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc704950>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 55135.89120769501, "timesteps_since_restore": 425000, "iterations_since_restore": 25, "perf": {"cpu_util_percent": 91.05381410256409, "ram_util_percent": 30.13602564102564}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 122731.06821363987, "episode_reward_min": 119590.04412822139, "episode_reward_mean": 121379.5060955943, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 570.9509082017903}, "policy_reward_mean": {"av": 389.0368785115202}, "custom_metrics": {"avg_speed_mean": 5.249783911876438, "avg_speed_min": 5.2033611483858655, "avg_speed_max": 5.3225131372713665, "avg_speed_avs_mean": 5.237593453404101, "avg_speed_avs_min": 5.184556477307651, "avg_speed_avs_max": 5.307571893949852, "avg_headway_avs_mean": 8.426199981539904, "avg_headway_avs_min": 7.504737906208802, "avg_headway_avs_max": 9.508498991868944, "std_headway_avs_mean": 8.529769190651258, "std_headway_avs_min": 7.525439602263746, "std_headway_avs_max": 9.731351155985735, "avg_accel_avs_mean": 0.7535459176924367, "avg_accel_avs_min": 0.7220324861930576, "avg_accel_avs_max": 0.7801118113856889, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 18.43121835568637, "avg_mpg_per_veh_min": 17.80992982922428, "avg_mpg_per_veh_max": 19.06925680663195, "num_cars_mean": 879.9162199999998, "num_cars_min": 872.349, "num_cars_max": 888.053, "avg_mpg_mean": 18.4106419102675, "avg_mpg_min": 17.799061635681692, "avg_mpg_max": 19.038847044777384}, "sampler_perf": {"mean_env_wait_ms": 1571.8084606091422, "mean_processing_ms": 417.4889489084408, "mean_inference_ms": 1.9181614152395616}, "off_policy_estimator": {}, "info": {"num_steps_trained": 29785088, "num_steps_sampled": 442000, "sample_time_ms": 2000851.737, "load_time_ms": 437.493, "grad_time_ms": 182000.549, "update_time_ms": 8.189, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3192.864501953125, "policy_loss": 0.00014212219684850425, "vf_loss": 3192.864501953125, "vf_explained_var": 0.26684290170669556, "vf_preds": 121.97586059570312, "kl": 0.00364744383841753, "entropy": 1.008287787437439, "entropy_coeff": 0.0, "advantages": 3.5307723464939045e-06, "rewards": 1.692839503288269}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 442000, "episodes_total": 442, "training_iteration": 26, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_17-12-06", "timestamp": 1599066726, "time_this_iter_s": 2203.2745101451874, "time_total_s": 57339.165717840195, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc71ea60>", "on_episode_step": ".on_episode_step at 0x7f3afc71e7b8>", "on_episode_end": ".on_episode_end at 0x7f3afc71e598>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc71e400>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 57339.165717840195, "timesteps_since_restore": 442000, "iterations_since_restore": 26, "perf": {"cpu_util_percent": 90.91283848359349, "ram_util_percent": 29.997833705001593}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 124330.72257844728, "episode_reward_min": 119590.04412822139, "episode_reward_mean": 121628.67173506452, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 570.9509082017903}, "policy_reward_mean": {"av": 389.124585645022}, "custom_metrics": {"avg_speed_mean": 5.247802497253589, "avg_speed_min": 5.187695590591489, "avg_speed_max": 5.3225131372713665, "avg_speed_avs_mean": 5.237479682895919, "avg_speed_avs_min": 5.184062215774663, "avg_speed_avs_max": 5.307571893949852, "avg_headway_avs_mean": 8.179586131917183, "avg_headway_avs_min": 7.504737906208802, "avg_headway_avs_max": 9.068419023724196, "std_headway_avs_mean": 8.280007833354409, "std_headway_avs_min": 7.525439602263746, "std_headway_avs_max": 9.293559002152781, "avg_accel_avs_mean": 0.7452168094384514, "avg_accel_avs_min": 0.7145120573182648, "avg_accel_avs_max": 0.7788599433545089, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 18.428779260334125, "avg_mpg_per_veh_min": 17.80992982922428, "avg_mpg_per_veh_max": 19.11471944073229, "num_cars_mean": 881.4176800000001, "num_cars_min": 872.349, "num_cars_max": 889.397, "avg_mpg_mean": 18.408586651426653, "avg_mpg_min": 17.799061635681692, "avg_mpg_max": 19.083629213585038}, "sampler_perf": {"mean_env_wait_ms": 1570.3211594075672, "mean_processing_ms": 418.8370053676574, "mean_inference_ms": 1.9152238703589008}, "off_policy_estimator": {}, "info": {"num_steps_trained": 30964096, "num_steps_sampled": 459000, "sample_time_ms": 2003005.319, "load_time_ms": 438.568, "grad_time_ms": 184416.506, "update_time_ms": 8.307, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3277.174560546875, "policy_loss": 0.00012727323337458074, "vf_loss": 3277.174560546875, "vf_explained_var": 0.247733473777771, "vf_preds": 124.2904281616211, "kl": 0.003697579726576805, "entropy": 1.0093320608139038, "entropy_coeff": 0.0, "advantages": 8.977374818641692e-06, "rewards": 1.7077006101608276}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 459000, "episodes_total": 459, "training_iteration": 27, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_17-49-04", "timestamp": 1599068944, "time_this_iter_s": 2217.9217932224274, "time_total_s": 59557.08751106262, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3b003b3a60>", "on_episode_step": ".on_episode_step at 0x7f3b003b3c80>", "on_episode_end": ".on_episode_end at 0x7f3b003b3d08>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc7047b8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 59557.08751106262, "timesteps_since_restore": 459000, "iterations_since_restore": 27, "perf": {"cpu_util_percent": 87.44498258942704, "ram_util_percent": 29.95163026274137}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 124330.72257844728, "episode_reward_min": 119590.04412822139, "episode_reward_mean": 121848.7900784133, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 573.6078875577048}, "policy_reward_mean": {"av": 389.15649477312536}, "custom_metrics": {"avg_speed_mean": 5.243510998047108, "avg_speed_min": 5.187695590591489, "avg_speed_max": 5.30111908120191, "avg_speed_avs_mean": 5.235390562305115, "avg_speed_avs_min": 5.184062215774663, "avg_speed_avs_max": 5.286989076410912, "avg_headway_avs_mean": 7.951825523063564, "avg_headway_avs_min": 7.2992371022988936, "avg_headway_avs_max": 8.948973465953946, "std_headway_avs_mean": 8.092214853443913, "std_headway_avs_min": 7.373246837076307, "std_headway_avs_max": 9.107133207455048, "avg_accel_avs_mean": 0.7337576596118213, "avg_accel_avs_min": 0.6937340082779999, "avg_accel_avs_max": 0.7709950502807315, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 18.43543082500484, "avg_mpg_per_veh_min": 17.80992982922428, "avg_mpg_per_veh_max": 19.11471944073229, "num_cars_mean": 882.9687999999999, "num_cars_min": 873.337, "num_cars_max": 890.516, "avg_mpg_mean": 18.416260148146907, "avg_mpg_min": 17.799061635681692, "avg_mpg_max": 19.083629213585038}, "sampler_perf": {"mean_env_wait_ms": 1569.4705562976096, "mean_processing_ms": 419.6760220427995, "mean_inference_ms": 1.9130465546089201}, "off_policy_estimator": {}, "info": {"num_steps_trained": 32145152, "num_steps_sampled": 476000, "sample_time_ms": 2007872.017, "load_time_ms": 439.296, "grad_time_ms": 185120.905, "update_time_ms": 8.561, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3250.275390625, "policy_loss": 0.0002296499296789989, "vf_loss": 3250.275146484375, "vf_explained_var": 0.2762335538864136, "vf_preds": 126.39842224121094, "kl": 0.0038180414121598005, "entropy": 0.9459571838378906, "entropy_coeff": 0.0, "advantages": -1.1275432854063183e-07, "rewards": 1.7000187635421753}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 476000, "episodes_total": 476, "training_iteration": 28, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_18-25-51", "timestamp": 1599071151, "time_this_iter_s": 2206.659130334854, "time_total_s": 61763.746641397476, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc71ec80>", "on_episode_step": ".on_episode_step at 0x7f3afc71ef28>", "on_episode_end": ".on_episode_end at 0x7f3afc71e840>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc71e158>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 61763.746641397476, "timesteps_since_restore": 476000, "iterations_since_restore": 28, "perf": {"cpu_util_percent": 90.88619153674834, "ram_util_percent": 29.96939230034998}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 124330.72257844728, "episode_reward_min": 119590.04412822139, "episode_reward_mean": 122077.74199022997, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 573.6078875577048}, "policy_reward_mean": {"av": 389.3281732052238}, "custom_metrics": {"avg_speed_mean": 5.240985998225279, "avg_speed_min": 5.187695590591489, "avg_speed_max": 5.289909917954839, "avg_speed_avs_mean": 5.234632085132127, "avg_speed_avs_min": 5.184062215774663, "avg_speed_avs_max": 5.286989076410912, "avg_headway_avs_mean": 7.773869241476814, "avg_headway_avs_min": 7.2992371022988936, "avg_headway_avs_max": 8.355506315214786, "std_headway_avs_mean": 7.931878292297991, "std_headway_avs_min": 7.373246837076307, "std_headway_avs_max": 8.750122397036833, "avg_accel_avs_mean": 0.7246604038329845, "avg_accel_avs_min": 0.6937340082779999, "avg_accel_avs_max": 0.7648547535073033, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 18.451347823511174, "avg_mpg_per_veh_min": 17.80992982922428, "avg_mpg_per_veh_max": 19.11471944073229, "num_cars_mean": 884.18777, "num_cars_min": 874.972, "num_cars_max": 890.516, "avg_mpg_mean": 18.432141147456836, "avg_mpg_min": 17.799061635681692, "avg_mpg_max": 19.083629213585038}, "sampler_perf": {"mean_env_wait_ms": 1568.8387616898192, "mean_processing_ms": 420.38395448735884, "mean_inference_ms": 1.9111598943480939}, "off_policy_estimator": {}, "info": {"num_steps_trained": 33324160, "num_steps_sampled": 493000, "sample_time_ms": 2010709.234, "load_time_ms": 443.955, "grad_time_ms": 185213.971, "update_time_ms": 8.822, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3391.67138671875, "policy_loss": 0.00019045834778808057, "vf_loss": 3391.671142578125, "vf_explained_var": 0.27340129017829895, "vf_preds": 126.71968078613281, "kl": 0.0034869289956986904, "entropy": 0.9500052332878113, "entropy_coeff": 0.0, "advantages": -7.270370588230435e-06, "rewards": 1.7037264108657837}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 493000, "episodes_total": 493, "training_iteration": 29, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_19-02-32", "timestamp": 1599073352, "time_this_iter_s": 2201.725091934204, "time_total_s": 63965.47173333168, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc71ed90>", "on_episode_step": ".on_episode_step at 0x7f3afc71e8c8>", "on_episode_end": ".on_episode_end at 0x7f3afc71e510>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc71e9d8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 63965.47173333168, "timesteps_since_restore": 493000, "iterations_since_restore": 29, "perf": {"cpu_util_percent": 90.98118622448979, "ram_util_percent": 29.964445153061224}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 124390.94442652664, "episode_reward_min": 120344.29201508216, "episode_reward_mean": 122495.32724055639, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 573.6078875577048}, "policy_reward_mean": {"av": 390.31139192122225}, "custom_metrics": {"avg_speed_mean": 5.236992963950956, "avg_speed_min": 5.187695590591489, "avg_speed_max": 5.289909917954839, "avg_speed_avs_mean": 5.231728203511361, "avg_speed_avs_min": 5.184062215774663, "avg_speed_avs_max": 5.286989076410912, "avg_headway_avs_mean": 7.719174018653543, "avg_headway_avs_min": 7.2992371022988936, "avg_headway_avs_max": 8.223118599554299, "std_headway_avs_mean": 7.85014092447081, "std_headway_avs_min": 7.373246837076307, "std_headway_avs_max": 8.750122397036833, "avg_accel_avs_mean": 0.7148681445271232, "avg_accel_avs_min": 0.6857971969956327, "avg_accel_avs_max": 0.7378400354689636, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 18.573521493906163, "avg_mpg_per_veh_min": 17.92414801497457, "avg_mpg_per_veh_max": 19.410678725484175, "num_cars_mean": 885.1491000000001, "num_cars_min": 874.972, "num_cars_max": 890.516, "avg_mpg_mean": 18.55317496655048, "avg_mpg_min": 17.916727364874234, "avg_mpg_max": 19.377142489931025}, "sampler_perf": {"mean_env_wait_ms": 1568.2582088748363, "mean_processing_ms": 421.01831695765964, "mean_inference_ms": 1.9093377691970315}, "off_policy_estimator": {}, "info": {"num_steps_trained": 34505472, "num_steps_sampled": 510000, "sample_time_ms": 2011747.771, "load_time_ms": 441.196, "grad_time_ms": 185436.18, "update_time_ms": 9.059, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3475.6533203125, "policy_loss": 0.00016533506277482957, "vf_loss": 3475.6533203125, "vf_explained_var": 0.26644623279571533, "vf_preds": 123.760009765625, "kl": 0.00380005594342947, "entropy": 0.9369567632675171, "entropy_coeff": 0.0, "advantages": 1.2514611626102123e-05, "rewards": 1.7119553089141846}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 510000, "episodes_total": 510, "training_iteration": 30, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_19-39-06", "timestamp": 1599075546, "time_this_iter_s": 2194.066158056259, "time_total_s": 66159.53789138794, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc704598>", "on_episode_step": ".on_episode_step at 0x7f3afc704510>", "on_episode_end": ".on_episode_end at 0x7f3afc704158>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc704268>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 66159.53789138794, "timesteps_since_restore": 510000, "iterations_since_restore": 30, "perf": {"cpu_util_percent": 90.47747199999999, "ram_util_percent": 29.964927999999997}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 125188.24234275785, "episode_reward_min": 120686.27121267465, "episode_reward_mean": 122971.67060549272, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 582.888923301689}, "policy_reward_mean": {"av": 391.62952422131445}, "custom_metrics": {"avg_speed_mean": 5.229839481313449, "avg_speed_min": 5.187695590591489, "avg_speed_max": 5.285992243996301, "avg_speed_avs_mean": 5.224838180687165, "avg_speed_avs_min": 5.182438266886283, "avg_speed_avs_max": 5.281637822463501, "avg_headway_avs_mean": 7.685794086275519, "avg_headway_avs_min": 7.2992371022988936, "avg_headway_avs_max": 8.105089190722754, "std_headway_avs_mean": 7.801876076771256, "std_headway_avs_min": 7.373246837076307, "std_headway_avs_max": 8.347590753221324, "avg_accel_avs_mean": 0.7076379504228933, "avg_accel_avs_min": 0.6812497317029559, "avg_accel_avs_max": 0.7378400354689636, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 18.7274598439024, "avg_mpg_per_veh_min": 17.92414801497457, "avg_mpg_per_veh_max": 19.504065361607744, "num_cars_mean": 886.4879799999998, "num_cars_min": 879.67, "num_cars_max": 891.883, "avg_mpg_mean": 18.70590815863976, "avg_mpg_min": 17.916727364874234, "avg_mpg_max": 19.461895475722645}, "sampler_perf": {"mean_env_wait_ms": 1567.7994838825125, "mean_processing_ms": 421.5963098849663, "mean_inference_ms": 1.9077309922350814}, "off_policy_estimator": {}, "info": {"num_steps_trained": 35691520, "num_steps_sampled": 527000, "sample_time_ms": 2013568.586, "load_time_ms": 440.505, "grad_time_ms": 185539.746, "update_time_ms": 9.01, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3388.895751953125, "policy_loss": 0.00017570544150657952, "vf_loss": 3388.8955078125, "vf_explained_var": 0.2739546597003937, "vf_preds": 122.86428833007812, "kl": 0.0036995429545640945, "entropy": 0.9457102417945862, "entropy_coeff": 0.0, "advantages": 6.689846031804336e-06, "rewards": 1.7147835493087769}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 527000, "episodes_total": 527, "training_iteration": 31, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_20-15-58", "timestamp": 1599077758, "time_this_iter_s": 2211.3758597373962, "time_total_s": 68370.91375112534, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afff9bb70>", "on_episode_step": ".on_episode_step at 0x7f3afff9b7b8>", "on_episode_end": ".on_episode_end at 0x7f3afff9b840>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afff9ba60>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 68370.91375112534, "timesteps_since_restore": 527000, "iterations_since_restore": 31, "perf": {"cpu_util_percent": 90.78238095238095, "ram_util_percent": 30.01326984126984}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 126200.17822708737, "episode_reward_min": 121694.39149465757, "episode_reward_mean": 123596.54999567899, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 582.888923301689}, "policy_reward_mean": {"av": 393.41911763330455}, "custom_metrics": {"avg_speed_mean": 5.220827869610649, "avg_speed_min": 5.163034701747512, "avg_speed_max": 5.285992243996301, "avg_speed_avs_mean": 5.215532271298234, "avg_speed_avs_min": 5.151416476179984, "avg_speed_avs_max": 5.281637822463501, "avg_headway_avs_mean": 7.70180329712421, "avg_headway_avs_min": 7.2992371022988936, "avg_headway_avs_max": 8.105089190722754, "std_headway_avs_mean": 7.793753950957409, "std_headway_avs_min": 7.373246837076307, "std_headway_avs_max": 8.347590753221324, "avg_accel_avs_mean": 0.6979118928110482, "avg_accel_avs_min": 0.6690422786653403, "avg_accel_avs_max": 0.7354794975022277, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 18.957167987720428, "avg_mpg_per_veh_min": 18.267203089776267, "avg_mpg_per_veh_max": 19.810354021066065, "num_cars_mean": 887.80395, "num_cars_min": 879.67, "num_cars_max": 895.077, "avg_mpg_mean": 18.933277110113778, "avg_mpg_min": 18.256890735160763, "avg_mpg_max": 19.776382710416}, "sampler_perf": {"mean_env_wait_ms": 1567.4326574902748, "mean_processing_ms": 422.12000204682477, "mean_inference_ms": 1.9062063067196606}, "off_policy_estimator": {}, "info": {"num_steps_trained": 36880256, "num_steps_sampled": 544000, "sample_time_ms": 2015419.832, "load_time_ms": 446.423, "grad_time_ms": 185536.108, "update_time_ms": 9.046, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3498.443115234375, "policy_loss": 0.00014267920050770044, "vf_loss": 3498.443115234375, "vf_explained_var": 0.25868481397628784, "vf_preds": 125.43049621582031, "kl": 0.0037389229983091354, "entropy": 0.9161908030509949, "entropy_coeff": 0.0, "advantages": 1.0394101082056295e-06, "rewards": 1.726150631904602}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 544000, "episodes_total": 544, "training_iteration": 32, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_20-52-43", "timestamp": 1599079963, "time_this_iter_s": 2205.252259016037, "time_total_s": 70576.16601014137, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc672e18>", "on_episode_step": ".on_episode_step at 0x7f3afc672ea0>", "on_episode_end": ".on_episode_end at 0x7f3afc672f28>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc6c0048>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 70576.16601014137, "timesteps_since_restore": 544000, "iterations_since_restore": 32, "perf": {"cpu_util_percent": 90.81372611464967, "ram_util_percent": 29.946496815286626}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 126200.17822708737, "episode_reward_min": 121694.39149465757, "episode_reward_mean": 123987.03658603324, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 582.888923301689}, "policy_reward_mean": {"av": 394.5364875772712}, "custom_metrics": {"avg_speed_mean": 5.213503041055033, "avg_speed_min": 5.141220070334577, "avg_speed_max": 5.271837320690593, "avg_speed_avs_mean": 5.208558081715175, "avg_speed_avs_min": 5.1307441867684025, "avg_speed_avs_max": 5.270733291402727, "avg_headway_avs_mean": 7.696039796633723, "avg_headway_avs_min": 7.2992371022988936, "avg_headway_avs_max": 8.002276934379692, "std_headway_avs_mean": 7.780767617747702, "std_headway_avs_min": 7.373246837076307, "std_headway_avs_max": 8.34222385095146, "avg_accel_avs_mean": 0.6892911546755215, "avg_accel_avs_min": 0.6584113632843284, "avg_accel_avs_max": 0.7166729248521908, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 19.112704297654776, "avg_mpg_per_veh_min": 18.267203089776267, "avg_mpg_per_veh_max": 19.997537356642074, "num_cars_mean": 888.9069699999999, "num_cars_min": 880.94, "num_cars_max": 895.756, "avg_mpg_mean": 19.088304485045363, "avg_mpg_min": 18.256890735160763, "avg_mpg_max": 19.96527752920455}, "sampler_perf": {"mean_env_wait_ms": 1566.9854641531465, "mean_processing_ms": 422.6463465679919, "mean_inference_ms": 1.904658554520791}, "off_policy_estimator": {}, "info": {"num_steps_trained": 38070272, "num_steps_sampled": 561000, "sample_time_ms": 2014903.101, "load_time_ms": 448.431, "grad_time_ms": 185445.863, "update_time_ms": 8.941, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3598.2978515625, "policy_loss": 0.00015083939069882035, "vf_loss": 3598.297607421875, "vf_explained_var": 0.26265496015548706, "vf_preds": 123.96232604980469, "kl": 0.0034953837748616934, "entropy": 0.9106456637382507, "entropy_coeff": 0.0, "advantages": 2.4670902121215477e-07, "rewards": 1.7248567342758179}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 561000, "episodes_total": 561, "training_iteration": 33, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_21-29-06", "timestamp": 1599082146, "time_this_iter_s": 2183.0348720550537, "time_total_s": 72759.20088219643, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc672620>", "on_episode_step": ".on_episode_step at 0x7f3afc672488>", "on_episode_end": ".on_episode_end at 0x7f3afc6721e0>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc6720d0>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 72759.20088219643, "timesteps_since_restore": 561000, "iterations_since_restore": 33, "perf": {"cpu_util_percent": 89.6850160771704, "ram_util_percent": 29.913408360128617}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 127566.63872234244, "episode_reward_min": 121769.74969255575, "episode_reward_mean": 124640.82471647034, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 582.888923301689}, "policy_reward_mean": {"av": 396.4528920018778}, "custom_metrics": {"avg_speed_mean": 5.203620250824782, "avg_speed_min": 5.141220070334577, "avg_speed_max": 5.264430611665306, "avg_speed_avs_mean": 5.198233813843762, "avg_speed_avs_min": 5.1307441867684025, "avg_speed_avs_max": 5.255553331308593, "avg_headway_avs_mean": 7.710379887425286, "avg_headway_avs_min": 7.445638596054316, "avg_headway_avs_max": 8.002276934379692, "std_headway_avs_mean": 7.768826971644338, "std_headway_avs_min": 7.376295869905541, "std_headway_avs_max": 8.34222385095146, "avg_accel_avs_mean": 0.6825700939438548, "avg_accel_avs_min": 0.6484102858407587, "avg_accel_avs_max": 0.7110838667977123, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 19.367032024527674, "avg_mpg_per_veh_min": 18.449487336613586, "avg_mpg_per_veh_max": 20.29072266577212, "num_cars_mean": 890.4412800000001, "num_cars_min": 881.653, "num_cars_max": 899.106, "avg_mpg_mean": 19.340052785722765, "avg_mpg_min": 18.424680511326986, "avg_mpg_max": 20.252744268032103}, "sampler_perf": {"mean_env_wait_ms": 1566.1816199959974, "mean_processing_ms": 423.49076387380586, "mean_inference_ms": 1.9027015787821366}, "off_policy_estimator": {}, "info": {"num_steps_trained": 39265280, "num_steps_sampled": 578000, "sample_time_ms": 2014832.377, "load_time_ms": 446.981, "grad_time_ms": 186019.888, "update_time_ms": 9.028, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3588.023681640625, "policy_loss": 0.00025394014664925635, "vf_loss": 3588.0234375, "vf_explained_var": 0.2649933993816376, "vf_preds": 121.72698211669922, "kl": 0.003263491205871105, "entropy": 0.9311535954475403, "entropy_coeff": 0.0, "advantages": 7.040128366497811e-06, "rewards": 1.732372760772705}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 578000, "episodes_total": 578, "training_iteration": 34, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_22-05-49", "timestamp": 1599084349, "time_this_iter_s": 2202.7956233024597, "time_total_s": 74961.99650549889, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3b000ef378>", "on_episode_step": ".on_episode_step at 0x7f3afc75f378>", "on_episode_end": ".on_episode_end at 0x7f3afc6722f0>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc672158>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 74961.99650549889, "timesteps_since_restore": 578000, "iterations_since_restore": 34, "perf": {"cpu_util_percent": 89.66579534587184, "ram_util_percent": 29.961938157475295}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 128794.03962553425, "episode_reward_min": 122837.20237837214, "episode_reward_mean": 125511.2637531185, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 582.888923301689}, "policy_reward_mean": {"av": 399.04385512707375}, "custom_metrics": {"avg_speed_mean": 5.187886659735443, "avg_speed_min": 5.066066491401064, "avg_speed_max": 5.254109405196808, "avg_speed_avs_mean": 5.1818930314434715, "avg_speed_avs_min": 5.0579742589099945, "avg_speed_avs_max": 5.255553331308593, "avg_headway_avs_mean": 7.75056796321135, "avg_headway_avs_min": 7.445638596054316, "avg_headway_avs_max": 8.212988769768497, "std_headway_avs_mean": 7.737579517060831, "std_headway_avs_min": 7.326938322486498, "std_headway_avs_max": 8.228212195790631, "avg_accel_avs_mean": 0.6740693341295796, "avg_accel_avs_min": 0.6462159650711371, "avg_accel_avs_max": 0.702239554348164, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 19.73363976105525, "avg_mpg_per_veh_min": 18.738466373183535, "avg_mpg_per_veh_max": 21.226519691204253, "num_cars_mean": 892.6234400000001, "num_cars_min": 883.572, "num_cars_max": 905.967, "avg_mpg_mean": 19.703915247734823, "avg_mpg_min": 18.716959020309712, "avg_mpg_max": 21.183841091020838}, "sampler_perf": {"mean_env_wait_ms": 1565.421281283345, "mean_processing_ms": 424.3233572547872, "mean_inference_ms": 1.9007191639953882}, "off_policy_estimator": {}, "info": {"num_steps_trained": 40465920, "num_steps_sampled": 595000, "sample_time_ms": 2016675.657, "load_time_ms": 450.117, "grad_time_ms": 187003.628, "update_time_ms": 8.718, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3669.1552734375, "policy_loss": 8.165251529135276e-06, "vf_loss": 3669.1552734375, "vf_explained_var": 0.23722681403160095, "vf_preds": 118.57110595703125, "kl": 0.004363152664154768, "entropy": 0.9357836246490479, "entropy_coeff": 0.0, "advantages": 2.3304193064177525e-07, "rewards": 1.7473126649856567}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 595000, "episodes_total": 595, "training_iteration": 35, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_22-42-49", "timestamp": 1599086569, "time_this_iter_s": 2219.294796705246, "time_total_s": 77181.29130220413, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc75ff28>", "on_episode_step": ".on_episode_step at 0x7f3afc75fe18>", "on_episode_end": ".on_episode_end at 0x7f3afc6726a8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc6720d0>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 77181.29130220413, "timesteps_since_restore": 595000, "iterations_since_restore": 35, "perf": {"cpu_util_percent": 87.87950664136622, "ram_util_percent": 30.125553447185325}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 129615.6113479407, "episode_reward_min": 123147.05016875512, "episode_reward_mean": 126418.83072301693, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 582.888923301689}, "policy_reward_mean": {"av": 401.76327058735427}, "custom_metrics": {"avg_speed_mean": 5.1719116007627495, "avg_speed_min": 5.066066491401064, "avg_speed_max": 5.24673945126706, "avg_speed_avs_mean": 5.165229344079595, "avg_speed_avs_min": 5.057392117262748, "avg_speed_avs_max": 5.234930703334773, "avg_headway_avs_mean": 7.800353190169335, "avg_headway_avs_min": 7.445638596054316, "avg_headway_avs_max": 8.365320593859444, "std_headway_avs_mean": 7.702567207633656, "std_headway_avs_min": 6.936925480628407, "std_headway_avs_max": 8.228212195790631, "avg_accel_avs_mean": 0.6648078338315067, "avg_accel_avs_min": 0.6365911130066566, "avg_accel_avs_max": 0.7012090275778491, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 20.109982470788133, "avg_mpg_per_veh_min": 18.920782131891194, "avg_mpg_per_veh_max": 21.496762040141792, "num_cars_mean": 894.94427, "num_cars_min": 885.326, "num_cars_max": 907.855, "avg_mpg_mean": 20.077478911424745, "avg_mpg_min": 18.903132348920163, "avg_mpg_max": 21.457495363037236}, "sampler_perf": {"mean_env_wait_ms": 1564.9682436953592, "mean_processing_ms": 425.09368125259857, "mean_inference_ms": 1.8990367014915521}, "off_policy_estimator": {}, "info": {"num_steps_trained": 41668480, "num_steps_sampled": 612000, "sample_time_ms": 2021066.028, "load_time_ms": 448.935, "grad_time_ms": 189773.744, "update_time_ms": 8.535, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3593.19970703125, "policy_loss": 0.0001052050938596949, "vf_loss": 3593.19970703125, "vf_explained_var": 0.2288905531167984, "vf_preds": 118.57646179199219, "kl": 0.0037153048906475306, "entropy": 0.9141941070556641, "entropy_coeff": 0.0, "advantages": 2.370672518736683e-06, "rewards": 1.757875919342041}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 612000, "episodes_total": 612, "training_iteration": 36, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_23-20-43", "timestamp": 1599088843, "time_this_iter_s": 2274.873335123062, "time_total_s": 79456.1646373272, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc672598>", "on_episode_step": ".on_episode_step at 0x7f3afc672730>", "on_episode_end": ".on_episode_end at 0x7f3afc6729d8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc672f28>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 79456.1646373272, "timesteps_since_restore": 612000, "iterations_since_restore": 36, "perf": {"cpu_util_percent": 87.46388888888889, "ram_util_percent": 30.11799382716049}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 129779.26984649904, "episode_reward_min": 124300.8474727546, "episode_reward_mean": 127146.28891585645, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 583.1792514757961}, "policy_reward_mean": {"av": 404.15222160157793}, "custom_metrics": {"avg_speed_mean": 5.161838449214363, "avg_speed_min": 5.066066491401064, "avg_speed_max": 5.242360127976204, "avg_speed_avs_mean": 5.154840285688116, "avg_speed_avs_min": 5.057392117262748, "avg_speed_avs_max": 5.2343658239440884, "avg_headway_avs_mean": 7.890473529344267, "avg_headway_avs_min": 7.445638596054316, "avg_headway_avs_max": 8.365320593859444, "std_headway_avs_mean": 7.700525944714884, "std_headway_avs_min": 6.936925480628407, "std_headway_avs_max": 8.228212195790631, "avg_accel_avs_mean": 0.6574571178938327, "avg_accel_avs_min": 0.6338242393569854, "avg_accel_avs_max": 0.6818832034677452, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 20.419092514000745, "avg_mpg_per_veh_min": 19.369492190020285, "avg_mpg_per_veh_max": 21.496762040141792, "num_cars_mean": 896.1838399999999, "num_cars_min": 886.996, "num_cars_max": 907.855, "avg_mpg_mean": 20.383342596527733, "avg_mpg_min": 19.354221890856195, "avg_mpg_max": 21.457495363037236}, "sampler_perf": {"mean_env_wait_ms": 1564.9215593975657, "mean_processing_ms": 425.66236934461887, "mean_inference_ms": 1.8977666013624668}, "off_policy_estimator": {}, "info": {"num_steps_trained": 42865536, "num_steps_sampled": 629000, "sample_time_ms": 2027183.453, "load_time_ms": 449.732, "grad_time_ms": 189739.619, "update_time_ms": 8.39, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3549.26708984375, "policy_loss": 0.0001722157612675801, "vf_loss": 3549.266845703125, "vf_explained_var": 0.2380685955286026, "vf_preds": 120.77685546875, "kl": 0.0037784678861498833, "entropy": 0.897018313407898, "entropy_coeff": 0.0, "advantages": 5.2726591093232855e-06, "rewards": 1.7606914043426514}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 629000, "episodes_total": 629, "training_iteration": 37, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-02_23-58-42", "timestamp": 1599091122, "time_this_iter_s": 2278.7554161548615, "time_total_s": 81734.92005348206, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc7041e0>", "on_episode_step": ".on_episode_step at 0x7f3afc704e18>", "on_episode_end": ".on_episode_end at 0x7f3afc704d90>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc704950>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 81734.92005348206, "timesteps_since_restore": 629000, "iterations_since_restore": 37, "perf": {"cpu_util_percent": 89.27220338983051, "ram_util_percent": 30.21537750385208}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 129779.26984649904, "episode_reward_min": 124417.40021593827, "episode_reward_mean": 127793.77814456534, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 583.1792514757961}, "policy_reward_mean": {"av": 406.3265973882081}, "custom_metrics": {"avg_speed_mean": 5.153966005949468, "avg_speed_min": 5.066066491401064, "avg_speed_max": 5.231230424320782, "avg_speed_avs_mean": 5.146375738994925, "avg_speed_avs_min": 5.057392117262748, "avg_speed_avs_max": 5.224363117482343, "avg_headway_avs_mean": 7.956466843245085, "avg_headway_avs_min": 7.445638596054316, "avg_headway_avs_max": 8.443067062266127, "std_headway_avs_mean": 7.710451403965798, "std_headway_avs_min": 6.936925480628407, "std_headway_avs_max": 8.144774205758747, "avg_accel_avs_mean": 0.6512087957679802, "avg_accel_avs_min": 0.6299329668150951, "avg_accel_avs_max": 0.6818832034677452, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 20.71334087698462, "avg_mpg_per_veh_min": 19.369492190020285, "avg_mpg_per_veh_max": 21.6072254161584, "num_cars_mean": 897.3857999999999, "num_cars_min": 888.798, "num_cars_max": 907.855, "avg_mpg_mean": 20.67437920655674, "avg_mpg_min": 19.354221890856195, "avg_mpg_max": 21.555988898140768}, "sampler_perf": {"mean_env_wait_ms": 1565.2957487977837, "mean_processing_ms": 425.87750006182205, "mean_inference_ms": 1.896989068544172}, "off_policy_estimator": {}, "info": {"num_steps_trained": 44064384, "num_steps_sampled": 646000, "sample_time_ms": 2028207.486, "load_time_ms": 452.381, "grad_time_ms": 190751.242, "update_time_ms": 8.346, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3581.599853515625, "policy_loss": 0.0002506137534510344, "vf_loss": 3581.599609375, "vf_explained_var": 0.2358924001455307, "vf_preds": 121.41724395751953, "kl": 0.003454490564763546, "entropy": 0.9003171324729919, "entropy_coeff": 0.0, "advantages": 1.4455558812187519e-06, "rewards": 1.765499234199524}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 646000, "episodes_total": 646, "training_iteration": 38, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_00-35-49", "timestamp": 1599093349, "time_this_iter_s": 2227.072345972061, "time_total_s": 83961.99239945412, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc71e840>", "on_episode_step": ".on_episode_step at 0x7f3afc71e0d0>", "on_episode_end": ".on_episode_end at 0x7f3afc71e8c8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc71ed90>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 83961.99239945412, "timesteps_since_restore": 646000, "iterations_since_restore": 38, "perf": {"cpu_util_percent": 89.57724550898203, "ram_util_percent": 30.192940434919635}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 130425.52410046857, "episode_reward_min": 125626.42288198728, "episode_reward_mean": 128473.10501120021, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 584.1825448018883}, "policy_reward_mean": {"av": 408.49953898632805}, "custom_metrics": {"avg_speed_mean": 5.144202748564135, "avg_speed_min": 5.066066491401064, "avg_speed_max": 5.199759806995536, "avg_speed_avs_mean": 5.135743515499842, "avg_speed_avs_min": 5.057392117262748, "avg_speed_avs_max": 5.195983022150622, "avg_headway_avs_mean": 8.027452755648495, "avg_headway_avs_min": 7.445638596054316, "avg_headway_avs_max": 8.443067062266127, "std_headway_avs_mean": 7.7023704144105185, "std_headway_avs_min": 6.936925480628407, "std_headway_avs_max": 8.143407060030697, "avg_accel_avs_mean": 0.6459930231295685, "avg_accel_avs_min": 0.6292579303494932, "avg_accel_avs_max": 0.6727586118092889, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 21.021788861009977, "avg_mpg_per_veh_min": 19.803093600640967, "avg_mpg_per_veh_max": 21.960885580155455, "num_cars_mean": 898.7142400000001, "num_cars_min": 892.711, "num_cars_max": 907.855, "avg_mpg_mean": 20.9791284797227, "avg_mpg_min": 19.771523929855007, "avg_mpg_max": 21.915078940966264}, "sampler_perf": {"mean_env_wait_ms": 1566.1516722957908, "mean_processing_ms": 425.6384667740569, "mean_inference_ms": 1.8967369210990435}, "off_policy_estimator": {}, "info": {"num_steps_trained": 45266688, "num_steps_sampled": 663000, "sample_time_ms": 2027666.3, "load_time_ms": 448.467, "grad_time_ms": 191287.391, "update_time_ms": 8.058, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3603.21435546875, "policy_loss": 0.00011254275887040421, "vf_loss": 3603.21435546875, "vf_explained_var": 0.21829184889793396, "vf_preds": 122.2362289428711, "kl": 0.003690088400617242, "entropy": 0.9132267832756042, "entropy_coeff": 0.0, "advantages": -7.983240720932372e-06, "rewards": 1.7645968198776245}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 663000, "episodes_total": 663, "training_iteration": 39, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_01-12-31", "timestamp": 1599095551, "time_this_iter_s": 2201.626844882965, "time_total_s": 86163.61924433708, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc6c0b70>", "on_episode_step": ".on_episode_step at 0x7f3afc6c0ae8>", "on_episode_end": ".on_episode_end at 0x7f3afc6c0a60>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc6c0048>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 86163.61924433708, "timesteps_since_restore": 663000, "iterations_since_restore": 39, "perf": {"cpu_util_percent": 88.57700892857143, "ram_util_percent": 30.172480867346938}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 130425.52410046857, "episode_reward_min": 126939.75204311596, "episode_reward_mean": 128826.49728220642, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 584.1825448018883}, "policy_reward_mean": {"av": 409.8317022402696}, "custom_metrics": {"avg_speed_mean": 5.14084692879937, "avg_speed_min": 5.066838065072587, "avg_speed_max": 5.194149705331481, "avg_speed_avs_mean": 5.13188011062235, "avg_speed_avs_min": 5.057392117262748, "avg_speed_avs_max": 5.188497513185267, "avg_headway_avs_mean": 8.145298113879877, "avg_headway_avs_min": 7.6553351142379915, "avg_headway_avs_max": 8.4707132349623, "std_headway_avs_mean": 7.743535987964687, "std_headway_avs_min": 6.936925480628407, "std_headway_avs_max": 8.165114532206738, "avg_accel_avs_mean": 0.641066538658704, "avg_accel_avs_min": 0.6259859073829209, "avg_accel_avs_max": 0.6649167646110146, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 21.20162784949143, "avg_mpg_per_veh_min": 20.61952615081596, "avg_mpg_per_veh_max": 21.960885580155455, "num_cars_mean": 898.9143200000002, "num_cars_min": 892.711, "num_cars_max": 907.855, "avg_mpg_mean": 21.156622812873294, "avg_mpg_min": 20.586519961123187, "avg_mpg_max": 21.915078940966264}, "sampler_perf": {"mean_env_wait_ms": 1567.391593912912, "mean_processing_ms": 424.99275711384047, "mean_inference_ms": 1.8970031569562231}, "off_policy_estimator": {}, "info": {"num_steps_trained": 46463872, "num_steps_sampled": 680000, "sample_time_ms": 2028210.034, "load_time_ms": 456.205, "grad_time_ms": 191656.335, "update_time_ms": 8.223, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3660.470947265625, "policy_loss": 9.506174683338031e-05, "vf_loss": 3660.470947265625, "vf_explained_var": 0.2356025129556656, "vf_preds": 124.25714874267578, "kl": 0.00436927005648613, "entropy": 0.862857460975647, "entropy_coeff": 0.0, "advantages": 7.018712949502515e-06, "rewards": 1.761422038078308}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 680000, "episodes_total": 680, "training_iteration": 40, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_01-49-14", "timestamp": 1599097754, "time_this_iter_s": 2203.2234675884247, "time_total_s": 88366.8427119255, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc6278c8>", "on_episode_step": ".on_episode_step at 0x7f3afc6c0ea0>", "on_episode_end": ".on_episode_end at 0x7f3afc6c0f28>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc704d90>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 88366.8427119255, "timesteps_since_restore": 680000, "iterations_since_restore": 40, "perf": {"cpu_util_percent": 89.4117272147865, "ram_util_percent": 30.218738049713192}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 130425.52410046857, "episode_reward_min": 127270.09477758752, "episode_reward_mean": 128979.27040396298, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 585.6744144430276}, "policy_reward_mean": {"av": 410.38299151727057}, "custom_metrics": {"avg_speed_mean": 5.144212754180582, "avg_speed_min": 5.084500864518809, "avg_speed_max": 5.197661632721393, "avg_speed_avs_mean": 5.135454128425529, "avg_speed_avs_min": 5.071949889535332, "avg_speed_avs_max": 5.188497513185267, "avg_headway_avs_mean": 8.183915354821444, "avg_headway_avs_min": 7.8647445448690565, "avg_headway_avs_max": 8.4707132349623, "std_headway_avs_mean": 7.790743738558112, "std_headway_avs_min": 7.287257002528471, "std_headway_avs_max": 8.233725682243122, "avg_accel_avs_mean": 0.6360297494509499, "avg_accel_avs_min": 0.6157936900054378, "avg_accel_avs_max": 0.6547499136451029, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 21.26928562595965, "avg_mpg_per_veh_min": 20.777927953598613, "avg_mpg_per_veh_max": 21.960885580155455, "num_cars_mean": 898.6928600000001, "num_cars_min": 892.956, "num_cars_max": 905.629, "avg_mpg_mean": 21.223323821427975, "avg_mpg_min": 20.744317199322495, "avg_mpg_max": 21.915078940966264}, "sampler_perf": {"mean_env_wait_ms": 1568.8871764394592, "mean_processing_ms": 423.98208492153594, "mean_inference_ms": 1.8976595816773898}, "off_policy_estimator": {}, "info": {"num_steps_trained": 47662080, "num_steps_sampled": 697000, "sample_time_ms": 2026883.524, "load_time_ms": 457.039, "grad_time_ms": 192148.172, "update_time_ms": 8.119, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3697.40966796875, "policy_loss": 0.0001334856206085533, "vf_loss": 3697.40966796875, "vf_explained_var": 0.24340806901454926, "vf_preds": 127.9268798828125, "kl": 0.003779648570343852, "entropy": 0.8437026739120483, "entropy_coeff": 0.0, "advantages": 6.956889251341636e-07, "rewards": 1.764633059501648}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 697000, "episodes_total": 697, "training_iteration": 41, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_02-25-58", "timestamp": 1599099958, "time_this_iter_s": 2203.094835996628, "time_total_s": 90569.93754792213, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5d6730>", "on_episode_step": ".on_episode_step at 0x7f3afc627378>", "on_episode_end": ".on_episode_end at 0x7f3afc627400>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc627488>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 90569.93754792213, "timesteps_since_restore": 697000, "iterations_since_restore": 41, "perf": {"cpu_util_percent": 88.94741873804972, "ram_util_percent": 30.158986615678774}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 131539.50049369386, "episode_reward_min": 127270.09477758752, "episode_reward_mean": 129211.63104540516, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 585.6744144430276}, "policy_reward_mean": {"av": 411.17464135371563}, "custom_metrics": {"avg_speed_mean": 5.139367753057703, "avg_speed_min": 5.067477594904065, "avg_speed_max": 5.197661632721393, "avg_speed_avs_mean": 5.130489741378653, "avg_speed_avs_min": 5.053539922653037, "avg_speed_avs_max": 5.188497513185267, "avg_headway_avs_mean": 8.200756052288895, "avg_headway_avs_min": 7.913493825998137, "avg_headway_avs_max": 8.4707132349623, "std_headway_avs_mean": 7.793941325021018, "std_headway_avs_min": 7.403870904177124, "std_headway_avs_max": 8.233725682243122, "avg_accel_avs_mean": 0.6316343781885966, "avg_accel_avs_min": 0.6111426758326176, "avg_accel_avs_max": 0.6508340592051053, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 21.394717754279235, "avg_mpg_per_veh_min": 20.830764692289673, "avg_mpg_per_veh_max": 22.243449638213562, "num_cars_mean": 899.33968, "num_cars_min": 892.956, "num_cars_max": 908.395, "avg_mpg_mean": 21.34808272768556, "avg_mpg_min": 20.789994654576525, "avg_mpg_max": 22.177151019168438}, "sampler_perf": {"mean_env_wait_ms": 1570.3420810859304, "mean_processing_ms": 422.78554218465183, "mean_inference_ms": 1.8983621239674207}, "off_policy_estimator": {}, "info": {"num_steps_trained": 48868864, "num_steps_sampled": 714000, "sample_time_ms": 2026738.325, "load_time_ms": 453.132, "grad_time_ms": 192480.557, "update_time_ms": 8.167, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3849.206298828125, "policy_loss": 0.00023260632588062435, "vf_loss": 3849.205810546875, "vf_explained_var": 0.20777136087417603, "vf_preds": 125.91239166259766, "kl": 0.003417183877900243, "entropy": 0.8692390322685242, "entropy_coeff": 0.0, "advantages": 1.1262736734352075e-06, "rewards": 1.769858956336975}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 714000, "episodes_total": 714, "training_iteration": 42, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_03-02-45", "timestamp": 1599102165, "time_this_iter_s": 2207.1588850021362, "time_total_s": 92777.09643292427, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc672378>", "on_episode_step": ".on_episode_step at 0x7f3afc672730>", "on_episode_end": ".on_episode_end at 0x7f3afc6726a8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc704e18>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 92777.09643292427, "timesteps_since_restore": 714000, "iterations_since_restore": 42, "perf": {"cpu_util_percent": 89.23000318167357, "ram_util_percent": 30.303118040089085}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 131539.50049369386, "episode_reward_min": 127270.09477758752, "episode_reward_mean": 129482.45939294608, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 594.2597593914293}, "policy_reward_mean": {"av": 411.9447041007447}, "custom_metrics": {"avg_speed_mean": 5.130791571360728, "avg_speed_min": 5.067477594904065, "avg_speed_max": 5.197661632721393, "avg_speed_avs_mean": 5.121859472947676, "avg_speed_avs_min": 5.053539922653037, "avg_speed_avs_max": 5.188497513185267, "avg_headway_avs_mean": 8.212433635620206, "avg_headway_avs_min": 7.913493825998137, "avg_headway_avs_max": 8.4707132349623, "std_headway_avs_mean": 7.783195861674899, "std_headway_avs_min": 7.425777200726339, "std_headway_avs_max": 8.233725682243122, "avg_accel_avs_mean": 0.6259418794265393, "avg_accel_avs_min": 0.6030914695438033, "avg_accel_avs_max": 0.6476386879715021, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 21.547313161768212, "avg_mpg_per_veh_min": 20.87871338445098, "avg_mpg_per_veh_max": 22.277192922363565, "num_cars_mean": 900.46364, "num_cars_min": 892.956, "num_cars_max": 908.395, "avg_mpg_mean": 21.49993666249247, "avg_mpg_min": 20.831576052597583, "avg_mpg_max": 22.232094885772682}, "sampler_perf": {"mean_env_wait_ms": 1571.5331228686514, "mean_processing_ms": 421.64625946595976, "mean_inference_ms": 1.8987978840391224}, "off_policy_estimator": {}, "info": {"num_steps_trained": 50075264, "num_steps_sampled": 731000, "sample_time_ms": 2029323.07, "load_time_ms": 454.498, "grad_time_ms": 193246.268, "update_time_ms": 8.156, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3829.728271484375, "policy_loss": 0.00013248230970930308, "vf_loss": 3829.728271484375, "vf_explained_var": 0.21502947807312012, "vf_preds": 126.88663482666016, "kl": 0.0037731314077973366, "entropy": 0.8468465209007263, "entropy_coeff": 0.0, "advantages": 1.3460372656481923e-06, "rewards": 1.7708806991577148}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 731000, "episodes_total": 731, "training_iteration": 43, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_03-39-41", "timestamp": 1599104381, "time_this_iter_s": 2216.5699334144592, "time_total_s": 94993.66636633873, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc64d158>", "on_episode_step": ".on_episode_step at 0x7f3afc64d1e0>", "on_episode_end": ".on_episode_end at 0x7f3afc64d268>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc64d2f0>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 94993.66636633873, "timesteps_since_restore": 731000, "iterations_since_restore": 43, "perf": {"cpu_util_percent": 89.43024065864472, "ram_util_percent": 30.24407853071564}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 131539.50049369386, "episode_reward_min": 127270.09477758752, "episode_reward_mean": 129724.2338620059, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 594.2597593914293}, "policy_reward_mean": {"av": 412.70077263387486}, "custom_metrics": {"avg_speed_mean": 5.12460777010054, "avg_speed_min": 5.067477594904065, "avg_speed_max": 5.197661632721393, "avg_speed_avs_mean": 5.115417367577722, "avg_speed_avs_min": 5.053539922653037, "avg_speed_avs_max": 5.188497513185267, "avg_headway_avs_mean": 8.242337190754652, "avg_headway_avs_min": 7.98792581292051, "avg_headway_avs_max": 8.65069877349245, "std_headway_avs_mean": 7.7689442666807444, "std_headway_avs_min": 7.425777200726339, "std_headway_avs_max": 8.233725682243122, "avg_accel_avs_mean": 0.6218813413342492, "avg_accel_avs_min": 0.6030914695438033, "avg_accel_avs_max": 0.6467542248883908, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 21.666343927659174, "avg_mpg_per_veh_min": 20.87871338445098, "avg_mpg_per_veh_max": 22.450013964165695, "num_cars_mean": 901.31424, "num_cars_min": 893.533, "num_cars_max": 908.58, "avg_mpg_mean": 21.618535362783796, "avg_mpg_min": 20.831576052597583, "avg_mpg_max": 22.370247185850342}, "sampler_perf": {"mean_env_wait_ms": 1572.315236584857, "mean_processing_ms": 420.75901912656553, "mean_inference_ms": 1.8988101612922776}, "off_policy_estimator": {}, "info": {"num_steps_trained": 51281408, "num_steps_sampled": 748000, "sample_time_ms": 2028025.648, "load_time_ms": 454.028, "grad_time_ms": 193422.306, "update_time_ms": 8.127, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3895.420166015625, "policy_loss": 0.00011032819020329043, "vf_loss": 3895.420166015625, "vf_explained_var": 0.20565973222255707, "vf_preds": 127.77589416503906, "kl": 0.0038273169193416834, "entropy": 0.8616589307785034, "entropy_coeff": 0.0, "advantages": 8.774289881330333e-07, "rewards": 1.7745171785354614}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 748000, "episodes_total": 748, "training_iteration": 44, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_04-16-13", "timestamp": 1599106573, "time_this_iter_s": 2191.6009843349457, "time_total_s": 97185.26735067368, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc64dc80>", "on_episode_step": ".on_episode_step at 0x7f3afc64d8c8>", "on_episode_end": ".on_episode_end at 0x7f3afc64dbf8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc64db70>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 97185.26735067368, "timesteps_since_restore": 748000, "iterations_since_restore": 44, "perf": {"cpu_util_percent": 88.13470041653316, "ram_util_percent": 30.24149311118231}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 131539.50049369386, "episode_reward_min": 127995.72300790377, "episode_reward_mean": 129830.87898743687, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 594.2597593914293}, "policy_reward_mean": {"av": 412.86929653194966}, "custom_metrics": {"avg_speed_mean": 5.126272240844535, "avg_speed_min": 5.067477594904065, "avg_speed_max": 5.197661632721393, "avg_speed_avs_mean": 5.1173777334186, "avg_speed_avs_min": 5.053539922653037, "avg_speed_avs_max": 5.188497513185267, "avg_headway_avs_mean": 8.198855799710627, "avg_headway_avs_min": 7.663841128184571, "avg_headway_avs_max": 8.65069877349245, "std_headway_avs_mean": 7.784751901183476, "std_headway_avs_min": 7.3630102579973284, "std_headway_avs_max": 8.233725682243122, "avg_accel_avs_mean": 0.617143884056757, "avg_accel_avs_min": 0.6030914695438033, "avg_accel_avs_max": 0.639436999560583, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 21.708091999963937, "avg_mpg_per_veh_min": 20.87871338445098, "avg_mpg_per_veh_max": 22.450013964165695, "num_cars_mean": 901.3573600000001, "num_cars_min": 893.533, "num_cars_max": 908.58, "avg_mpg_mean": 21.65991816756873, "avg_mpg_min": 20.831576052597583, "avg_mpg_max": 22.370247185850342}, "sampler_perf": {"mean_env_wait_ms": 1572.706236467909, "mean_processing_ms": 420.25445198928566, "mean_inference_ms": 1.8984212820564137}, "off_policy_estimator": {}, "info": {"num_steps_trained": 52484096, "num_steps_sampled": 765000, "sample_time_ms": 2026267.617, "load_time_ms": 453.219, "grad_time_ms": 193793.204, "update_time_ms": 8.145, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3781.922607421875, "policy_loss": 0.00020981520356144756, "vf_loss": 3781.922119140625, "vf_explained_var": 0.24387028813362122, "vf_preds": 131.57662963867188, "kl": 0.003474919591099024, "entropy": 0.8440508246421814, "entropy_coeff": 0.0, "advantages": -9.696494089439511e-06, "rewards": 1.7705934047698975}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 765000, "episodes_total": 765, "training_iteration": 45, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_04-52-59", "timestamp": 1599108779, "time_this_iter_s": 2205.4044427871704, "time_total_s": 99390.67179346085, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc672510>", "on_episode_step": ".on_episode_step at 0x7f3afc672400>", "on_episode_end": ".on_episode_end at 0x7f3afc672268>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc672598>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 99390.67179346085, "timesteps_since_restore": 765000, "iterations_since_restore": 45, "perf": {"cpu_util_percent": 88.99471674092935, "ram_util_percent": 30.370241884150218}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 131539.50049369386, "episode_reward_min": 128383.9157172316, "episode_reward_mean": 130014.82595859177, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 594.2597593914293}, "policy_reward_mean": {"av": 413.2045954507922}, "custom_metrics": {"avg_speed_mean": 5.126435481276164, "avg_speed_min": 5.067477594904065, "avg_speed_max": 5.202195132452334, "avg_speed_avs_mean": 5.1172819546700286, "avg_speed_avs_min": 5.053539922653037, "avg_speed_avs_max": 5.192214449737887, "avg_headway_avs_mean": 8.10990778237299, "avg_headway_avs_min": 7.558321397402569, "avg_headway_avs_max": 8.65069877349245, "std_headway_avs_mean": 7.769108225742383, "std_headway_avs_min": 7.3630102579973284, "std_headway_avs_max": 8.233725682243122, "avg_accel_avs_mean": 0.6146471437226387, "avg_accel_avs_min": 0.6030914695438033, "avg_accel_avs_max": 0.6293190286694867, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 21.774058621683853, "avg_mpg_per_veh_min": 20.955420216627804, "avg_mpg_per_veh_max": 22.450013964165695, "num_cars_mean": 901.77304, "num_cars_min": 894.276, "num_cars_max": 908.58, "avg_mpg_mean": 21.725089701293953, "avg_mpg_min": 20.919665060892775, "avg_mpg_max": 22.370247185850342}, "sampler_perf": {"mean_env_wait_ms": 1572.6829012245437, "mean_processing_ms": 420.12038811201876, "mean_inference_ms": 1.8975548677610683}, "off_policy_estimator": {}, "info": {"num_steps_trained": 53686912, "num_steps_sampled": 782000, "sample_time_ms": 2020512.609, "load_time_ms": 458.497, "grad_time_ms": 191399.999, "update_time_ms": 8.241, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3741.542724609375, "policy_loss": 0.0002031558979069814, "vf_loss": 3741.542236328125, "vf_explained_var": 0.2628231346607208, "vf_preds": 131.44454956054688, "kl": 0.0036187872756272554, "entropy": 0.8548306822776794, "entropy_coeff": 0.0, "advantages": 7.390439805021742e-06, "rewards": 1.7690833806991577}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 782000, "episodes_total": 782, "training_iteration": 46, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_05-29-32", "timestamp": 1599110972, "time_this_iter_s": 2193.454878807068, "time_total_s": 101584.12667226791, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3b003b3730>", "on_episode_step": ".on_episode_step at 0x7f3afc6c0b70>", "on_episode_end": ".on_episode_end at 0x7f3afc6c0ae8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc6c01e0>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 101584.12667226791, "timesteps_since_restore": 782000, "iterations_since_restore": 46, "perf": {"cpu_util_percent": 88.46056338028168, "ram_util_percent": 30.341549295774648}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 131539.50049369386, "episode_reward_min": 128127.89232717673, "episode_reward_mean": 130006.30417171067, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 594.2597593914293}, "policy_reward_mean": {"av": 413.05936382954394}, "custom_metrics": {"avg_speed_mean": 5.128342338489029, "avg_speed_min": 5.067477594904065, "avg_speed_max": 5.202195132452334, "avg_speed_avs_mean": 5.119226796839182, "avg_speed_avs_min": 5.053539922653037, "avg_speed_avs_max": 5.19235862636219, "avg_headway_avs_mean": 8.066965661673596, "avg_headway_avs_min": 7.558321397402569, "avg_headway_avs_max": 8.65069877349245, "std_headway_avs_mean": 7.782440858838842, "std_headway_avs_min": 7.3630102579973284, "std_headway_avs_max": 8.167724802547092, "avg_accel_avs_mean": 0.6134772433277589, "avg_accel_avs_min": 0.6030914695438033, "avg_accel_avs_max": 0.6276657768867792, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 21.765717813745958, "avg_mpg_per_veh_min": 21.048815849587438, "avg_mpg_per_veh_max": 22.450013964165695, "num_cars_mean": 901.66687, "num_cars_min": 892.375, "num_cars_max": 908.58, "avg_mpg_mean": 21.716629993340383, "avg_mpg_min": 21.012674910387005, "avg_mpg_max": 22.370247185850342}, "sampler_perf": {"mean_env_wait_ms": 1572.2800042294796, "mean_processing_ms": 420.3231484369191, "mean_inference_ms": 1.8962756316119935}, "off_policy_estimator": {}, "info": {"num_steps_trained": 54886272, "num_steps_sampled": 799000, "sample_time_ms": 2011954.481, "load_time_ms": 464.928, "grad_time_ms": 189778.604, "update_time_ms": 8.437, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3786.3828125, "policy_loss": 0.0002415099588688463, "vf_loss": 3786.382568359375, "vf_explained_var": 0.26367419958114624, "vf_preds": 134.47877502441406, "kl": 0.0036682740319520235, "entropy": 0.8165897130966187, "entropy_coeff": 0.0, "advantages": 7.178078703873325e-06, "rewards": 1.7650591135025024}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 799000, "episodes_total": 799, "training_iteration": 47, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_06-05-49", "timestamp": 1599113149, "time_this_iter_s": 2177.123923778534, "time_total_s": 103761.25059604645, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc6c0bf8>", "on_episode_step": ".on_episode_step at 0x7f3afc6c00d0>", "on_episode_end": ".on_episode_end at 0x7f3afc6c0158>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc6c0268>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 103761.25059604645, "timesteps_since_restore": 799000, "iterations_since_restore": 47, "perf": {"cpu_util_percent": 88.64230893260238, "ram_util_percent": 30.220702999032568}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 131295.67628728543, "episode_reward_min": 128127.89232717673, "episode_reward_mean": 129804.57905032401, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 594.2597593914293}, "policy_reward_mean": {"av": 412.104194076843}, "custom_metrics": {"avg_speed_mean": 5.137104521625072, "avg_speed_min": 5.069038521373219, "avg_speed_max": 5.205045544903473, "avg_speed_avs_mean": 5.128858155862238, "avg_speed_avs_min": 5.056012740584001, "avg_speed_avs_max": 5.202061617521353, "avg_headway_avs_mean": 7.978659200240984, "avg_headway_avs_min": 7.4840114984781945, "avg_headway_avs_max": 8.65069877349245, "std_headway_avs_mean": 7.794095752889282, "std_headway_avs_min": 7.3630102579973284, "std_headway_avs_max": 8.269252247423111, "avg_accel_avs_mean": 0.6120928910967034, "avg_accel_avs_min": 0.597924869984454, "avg_accel_avs_max": 0.6276657768867792, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 21.644506772754735, "avg_mpg_per_veh_min": 20.8029439676382, "avg_mpg_per_veh_max": 22.450013964165695, "num_cars_mean": 900.9002700000002, "num_cars_min": 892.375, "num_cars_max": 908.58, "avg_mpg_mean": 21.595453719180263, "avg_mpg_min": 20.772511483175474, "avg_mpg_max": 22.370247185850342}, "sampler_perf": {"mean_env_wait_ms": 1571.5602539329898, "mean_processing_ms": 420.67540037034996, "mean_inference_ms": 1.8946937636337395}, "off_policy_estimator": {}, "info": {"num_steps_trained": 56087808, "num_steps_sampled": 816000, "sample_time_ms": 2006242.501, "load_time_ms": 468.971, "grad_time_ms": 188967.9, "update_time_ms": 8.335, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3851.01904296875, "policy_loss": 0.00011706907389452681, "vf_loss": 3851.0185546875, "vf_explained_var": 0.2785595953464508, "vf_preds": 134.59396362304688, "kl": 0.00461692875251174, "entropy": 0.7678019404411316, "entropy_coeff": 0.0, "advantages": -4.195390374661656e-06, "rewards": 1.7614597082138062}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 816000, "episodes_total": 816, "training_iteration": 48, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_06-41-51", "timestamp": 1599115311, "time_this_iter_s": 2161.832245349884, "time_total_s": 105923.08284139633, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc627f28>", "on_episode_step": ".on_episode_step at 0x7f3afc627048>", "on_episode_end": ".on_episode_end at 0x7f3afc6271e0>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc6270d0>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 105923.08284139633, "timesteps_since_restore": 816000, "iterations_since_restore": 48, "perf": {"cpu_util_percent": 87.12159090909091, "ram_util_percent": 30.154480519480515}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 131295.67628728543, "episode_reward_min": 128127.89232717673, "episode_reward_mean": 129735.44724051902, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 595.0121480796811}, "policy_reward_mean": {"av": 411.59723109301694}, "custom_metrics": {"avg_speed_mean": 5.146851275328521, "avg_speed_min": 5.069504243882823, "avg_speed_max": 5.205045544903473, "avg_speed_avs_mean": 5.139281318563175, "avg_speed_avs_min": 5.056204897879958, "avg_speed_avs_max": 5.202061617521353, "avg_headway_avs_mean": 7.9031098095505525, "avg_headway_avs_min": 7.4840114984781945, "avg_headway_avs_max": 8.65069877349245, "std_headway_avs_mean": 7.804198177278607, "std_headway_avs_min": 7.316072347348107, "std_headway_avs_max": 8.269252247423111, "avg_accel_avs_mean": 0.6097448109443772, "avg_accel_avs_min": 0.5857784961209814, "avg_accel_avs_max": 0.6276657768867792, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 21.56394020639711, "avg_mpg_per_veh_min": 20.8029439676382, "avg_mpg_per_veh_max": 22.450013964165695, "num_cars_mean": 900.01459, "num_cars_min": 892.224, "num_cars_max": 908.58, "avg_mpg_mean": 21.515418258605802, "avg_mpg_min": 20.772511483175474, "avg_mpg_max": 22.370247185850342}, "sampler_perf": {"mean_env_wait_ms": 1570.6483831610378, "mean_processing_ms": 421.1219859733862, "mean_inference_ms": 1.8929396610959457}, "off_policy_estimator": {}, "info": {"num_steps_trained": 57288320, "num_steps_sampled": 833000, "sample_time_ms": 2004874.344, "load_time_ms": 476.436, "grad_time_ms": 188809.736, "update_time_ms": 8.307, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3933.86083984375, "policy_loss": 0.00015742274990770966, "vf_loss": 3933.86083984375, "vf_explained_var": 0.288444846868515, "vf_preds": 134.50607299804688, "kl": 0.00411098962649703, "entropy": 0.7325291037559509, "entropy_coeff": 0.0, "advantages": -3.5699092677532462e-06, "rewards": 1.7727320194244385}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 833000, "episodes_total": 833, "training_iteration": 49, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_07-18-18", "timestamp": 1599117498, "time_this_iter_s": 2186.4698650836945, "time_total_s": 108109.55270648003, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc597378>", "on_episode_step": ".on_episode_step at 0x7f3afc597400>", "on_episode_end": ".on_episode_end at 0x7f3afc597488>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc597510>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 108109.55270648003, "timesteps_since_restore": 833000, "iterations_since_restore": 49, "perf": {"cpu_util_percent": 88.16763005780346, "ram_util_percent": 30.126043673731534}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 130604.95105753836, "episode_reward_min": 128127.89232717673, "episode_reward_mean": 129510.612081153, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 595.0121480796811}, "policy_reward_mean": {"av": 410.66243485795405}, "custom_metrics": {"avg_speed_mean": 5.15659009745251, "avg_speed_min": 5.097292311151273, "avg_speed_max": 5.205045544903473, "avg_speed_avs_mean": 5.150001787043116, "avg_speed_avs_min": 5.093759733485258, "avg_speed_avs_max": 5.202061617521353, "avg_headway_avs_mean": 7.844153218553271, "avg_headway_avs_min": 7.4840114984781945, "avg_headway_avs_max": 8.196379075777283, "std_headway_avs_mean": 7.830647472019974, "std_headway_avs_min": 7.316072347348107, "std_headway_avs_max": 8.317897945748118, "avg_accel_avs_mean": 0.6087449289716381, "avg_accel_avs_min": 0.5857784961209814, "avg_accel_avs_max": 0.6276657768867792, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 21.434362410481206, "avg_mpg_per_veh_min": 20.8029439676382, "avg_mpg_per_veh_max": 22.035985447456174, "num_cars_mean": 898.81606, "num_cars_min": 892.224, "num_cars_max": 906.001, "avg_mpg_mean": 21.385785672093224, "avg_mpg_min": 20.772511483175474, "avg_mpg_max": 21.98032285093139}, "sampler_perf": {"mean_env_wait_ms": 1569.5839902691832, "mean_processing_ms": 421.6253118258597, "mean_inference_ms": 1.8910546923198583}, "off_policy_estimator": {}, "info": {"num_steps_trained": 58485888, "num_steps_sampled": 850000, "sample_time_ms": 2001172.556, "load_time_ms": 474.688, "grad_time_ms": 188831.632, "update_time_ms": 7.959, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4043.809814453125, "policy_loss": 0.0002674559655133635, "vf_loss": 4043.809326171875, "vf_explained_var": 0.2861000597476959, "vf_preds": 134.4940185546875, "kl": 0.003665389958769083, "entropy": 0.7418743371963501, "entropy_coeff": 0.0, "advantages": -7.344476102844055e-07, "rewards": 1.7689573764801025}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 850000, "episodes_total": 850, "training_iteration": 50, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_07-54-24", "timestamp": 1599119664, "time_this_iter_s": 2166.413719892502, "time_total_s": 110275.96642637253, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc672c80>", "on_episode_step": ".on_episode_step at 0x7f3afc672840>", "on_episode_end": ".on_episode_end at 0x7f3afc6727b8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc672950>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 110275.96642637253, "timesteps_since_restore": 850000, "iterations_since_restore": 50, "perf": {"cpu_util_percent": 87.08227478937137, "ram_util_percent": 30.295593000648086}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 130826.35477954037, "episode_reward_min": 128127.89232717673, "episode_reward_mean": 129439.01257549565, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 595.0121480796811}, "policy_reward_mean": {"av": 410.3573299162907}, "custom_metrics": {"avg_speed_mean": 5.1601796109489975, "avg_speed_min": 5.108995717376061, "avg_speed_max": 5.207234617148101, "avg_speed_avs_mean": 5.154298621386108, "avg_speed_avs_min": 5.098864299012014, "avg_speed_avs_max": 5.210072492817937, "avg_headway_avs_mean": 7.810022599446306, "avg_headway_avs_min": 7.4840114984781945, "avg_headway_avs_max": 8.196379075777283, "std_headway_avs_mean": 7.835514662525113, "std_headway_avs_min": 7.316072347348107, "std_headway_avs_max": 8.317897945748118, "avg_accel_avs_mean": 0.6080720038027857, "avg_accel_avs_min": 0.5857784961209814, "avg_accel_avs_max": 0.6276657768867792, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 21.368839289173867, "avg_mpg_per_veh_min": 20.8029439676382, "avg_mpg_per_veh_max": 21.775024888975004, "num_cars_mean": 898.5576800000001, "num_cars_min": 892.224, "num_cars_max": 904.037, "avg_mpg_mean": 21.31980243298241, "avg_mpg_min": 20.772511483175474, "avg_mpg_max": 21.728563206129884}, "sampler_perf": {"mean_env_wait_ms": 1568.4526866633935, "mean_processing_ms": 422.1497006345081, "mean_inference_ms": 1.8891618966761763}, "off_policy_estimator": {}, "info": {"num_steps_trained": 59685248, "num_steps_sampled": 867000, "sample_time_ms": 1999646.31, "load_time_ms": 474.742, "grad_time_ms": 188409.582, "update_time_ms": 8.081, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4070.08447265625, "policy_loss": 0.00015268626157194376, "vf_loss": 4070.08447265625, "vf_explained_var": 0.29546695947647095, "vf_preds": 132.74261474609375, "kl": 0.004049192648380995, "entropy": 0.763662576675415, "entropy_coeff": 0.0, "advantages": -3.850568191410275e-06, "rewards": 1.7695066928863525}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 867000, "episodes_total": 867, "training_iteration": 51, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_08-30-48", "timestamp": 1599121848, "time_this_iter_s": 2183.5901391506195, "time_total_s": 112459.55656552315, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc627730>", "on_episode_step": ".on_episode_step at 0x7f3afc627158>", "on_episode_end": ".on_episode_end at 0x7f3afc627ae8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc627840>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 112459.55656552315, "timesteps_since_restore": 867000, "iterations_since_restore": 51, "perf": {"cpu_util_percent": 88.04770170363228, "ram_util_percent": 30.237126325940217}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 130826.35477954037, "episode_reward_min": 128127.89232717673, "episode_reward_mean": 129390.47349108125, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 595.0121480796811}, "policy_reward_mean": {"av": 410.1384350547774}, "custom_metrics": {"avg_speed_mean": 5.164019423828891, "avg_speed_min": 5.1190543308703385, "avg_speed_max": 5.212584993868271, "avg_speed_avs_mean": 5.158642218906359, "avg_speed_avs_min": 5.109645593018678, "avg_speed_avs_max": 5.210072492817937, "avg_headway_avs_mean": 7.81186841600217, "avg_headway_avs_min": 7.476609278807222, "avg_headway_avs_max": 8.196379075777283, "std_headway_avs_mean": 7.8637238931234625, "std_headway_avs_min": 7.316072347348107, "std_headway_avs_max": 8.415161995678456, "avg_accel_avs_mean": 0.6057518174157498, "avg_accel_avs_min": 0.5857784961209814, "avg_accel_avs_max": 0.6215877487903267, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 21.340350407760454, "avg_mpg_per_veh_min": 20.8029439676382, "avg_mpg_per_veh_max": 21.786157252366014, "num_cars_mean": 898.09076, "num_cars_min": 891.861, "num_cars_max": 904.937, "avg_mpg_mean": 21.290701149470515, "avg_mpg_min": 20.772511483175474, "avg_mpg_max": 21.73304808470705}, "sampler_perf": {"mean_env_wait_ms": 1567.2439614628988, "mean_processing_ms": 422.6591466390717, "mean_inference_ms": 1.8872245799618048}, "off_policy_estimator": {}, "info": {"num_steps_trained": 60884224, "num_steps_sampled": 884000, "sample_time_ms": 1995299.265, "load_time_ms": 478.217, "grad_time_ms": 188555.564, "update_time_ms": 7.686, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4030.29833984375, "policy_loss": 0.00020191390649415553, "vf_loss": 4030.2978515625, "vf_explained_var": 0.3006249666213989, "vf_preds": 131.42747497558594, "kl": 0.003931706305593252, "entropy": 0.740182638168335, "entropy_coeff": 0.0, "advantages": -3.7170673294895096e-06, "rewards": 1.7688014507293701}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 884000, "episodes_total": 884, "training_iteration": 52, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_09-06-53", "timestamp": 1599124013, "time_this_iter_s": 2165.1365649700165, "time_total_s": 114624.69313049316, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5f61e0>", "on_episode_step": ".on_episode_step at 0x7f3afc5f6268>", "on_episode_end": ".on_episode_end at 0x7f3afc5f62f0>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5f6378>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 114624.69313049316, "timesteps_since_restore": 884000, "iterations_since_restore": 52, "perf": {"cpu_util_percent": 87.01523994811933, "ram_util_percent": 30.287321660181583}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 130826.35477954037, "episode_reward_min": 128140.66366292919, "episode_reward_mean": 129428.56395777293, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 595.0121480796811}, "policy_reward_mean": {"av": 410.2071626450712}, "custom_metrics": {"avg_speed_mean": 5.166958418430881, "avg_speed_min": 5.1190543308703385, "avg_speed_max": 5.212584993868271, "avg_speed_avs_mean": 5.162214023828533, "avg_speed_avs_min": 5.110082840997002, "avg_speed_avs_max": 5.210072492817937, "avg_headway_avs_mean": 7.789438096676507, "avg_headway_avs_min": 7.476609278807222, "avg_headway_avs_max": 8.196379075777283, "std_headway_avs_mean": 7.883313593064738, "std_headway_avs_min": 7.316072347348107, "std_headway_avs_max": 8.415161995678456, "avg_accel_avs_mean": 0.6038020342392341, "avg_accel_avs_min": 0.5857784961209814, "avg_accel_avs_max": 0.6171499444476003, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 21.34581065795263, "avg_mpg_per_veh_min": 20.8029439676382, "avg_mpg_per_veh_max": 21.786157252366014, "num_cars_mean": 897.86549, "num_cars_min": 891.861, "num_cars_max": 904.937, "avg_mpg_mean": 21.294994746186745, "avg_mpg_min": 20.772511483175474, "avg_mpg_max": 21.73304808470705}, "sampler_perf": {"mean_env_wait_ms": 1566.027194727388, "mean_processing_ms": 423.1508481217835, "mean_inference_ms": 1.885315565846764}, "off_policy_estimator": {}, "info": {"num_steps_trained": 62082432, "num_steps_sampled": 901000, "sample_time_ms": 1990465.968, "load_time_ms": 476.534, "grad_time_ms": 188164.692, "update_time_ms": 7.827, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3972.662841796875, "policy_loss": 0.00019061428611166775, "vf_loss": 3972.662841796875, "vf_explained_var": 0.3068484961986542, "vf_preds": 130.474853515625, "kl": 0.003959549590945244, "entropy": 0.7415890097618103, "entropy_coeff": 0.0, "advantages": -2.224016952823149e-06, "rewards": 1.7699270248413086}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 901000, "episodes_total": 901, "training_iteration": 53, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_09-42-57", "timestamp": 1599126177, "time_this_iter_s": 2164.3159272670746, "time_total_s": 116789.00905776024, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc627950>", "on_episode_step": ".on_episode_step at 0x7f3afc627268>", "on_episode_end": ".on_episode_end at 0x7f3afc627598>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc627bf8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 116789.00905776024, "timesteps_since_restore": 901000, "iterations_since_restore": 53, "perf": {"cpu_util_percent": 87.77054816736944, "ram_util_percent": 30.28897178073305}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 131408.51983569536, "episode_reward_min": 128325.92630306997, "episode_reward_mean": 129623.67466150252, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 595.0121480796811}, "policy_reward_mean": {"av": 410.79950136750483}, "custom_metrics": {"avg_speed_mean": 5.164791745890894, "avg_speed_min": 5.107621111993096, "avg_speed_max": 5.212584993868271, "avg_speed_avs_mean": 5.1598188479453775, "avg_speed_avs_min": 5.1057914687000014, "avg_speed_avs_max": 5.210072492817937, "avg_headway_avs_mean": 7.789025700667363, "avg_headway_avs_min": 7.409424394795882, "avg_headway_avs_max": 8.196379075777283, "std_headway_avs_mean": 7.904804162177317, "std_headway_avs_min": 7.316072347348107, "std_headway_avs_max": 8.415161995678456, "avg_accel_avs_mean": 0.6021687642928808, "avg_accel_avs_min": 0.5857784961209814, "avg_accel_avs_max": 0.6171499444476003, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 21.434312424657424, "avg_mpg_per_veh_min": 20.913283623020863, "avg_mpg_per_veh_max": 22.17716838665946, "num_cars_mean": 898.14185, "num_cars_min": 891.861, "num_cars_max": 908.074, "avg_mpg_mean": 21.381381771100678, "avg_mpg_min": 20.873835366909578, "avg_mpg_max": 22.097555325818632}, "sampler_perf": {"mean_env_wait_ms": 1564.8684633961896, "mean_processing_ms": 423.66029662548215, "mean_inference_ms": 1.8834737321749293}, "off_policy_estimator": {}, "info": {"num_steps_trained": 63286656, "num_steps_sampled": 918000, "sample_time_ms": 1990254.058, "load_time_ms": 485.607, "grad_time_ms": 187936.319, "update_time_ms": 7.697, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3970.535400390625, "policy_loss": 0.00018114002887159586, "vf_loss": 3970.534912109375, "vf_explained_var": 0.29317644238471985, "vf_preds": 127.98394775390625, "kl": 0.0041136108338832855, "entropy": 0.7577577829360962, "entropy_coeff": 0.0, "advantages": -9.219459570886102e-06, "rewards": 1.774856448173523}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 918000, "episodes_total": 918, "training_iteration": 54, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_10-19-25", "timestamp": 1599128365, "time_this_iter_s": 2187.2432782649994, "time_total_s": 118976.25233602524, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc672d90>", "on_episode_step": ".on_episode_step at 0x7f3afc672268>", "on_episode_end": ".on_episode_end at 0x7f3afc672730>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afff9b6a8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 118976.25233602524, "timesteps_since_restore": 918000, "iterations_since_restore": 54, "perf": {"cpu_util_percent": 87.72199036918138, "ram_util_percent": 30.402632423756017}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 131805.65075938756, "episode_reward_min": 128325.92630306997, "episode_reward_mean": 129830.94412006825, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 589.9095307561965}, "policy_reward_mean": {"av": 411.4302957284453}, "custom_metrics": {"avg_speed_mean": 5.160620282438952, "avg_speed_min": 5.09923953951576, "avg_speed_max": 5.212584993868271, "avg_speed_avs_mean": 5.15560010810226, "avg_speed_avs_min": 5.095013043611431, "avg_speed_avs_max": 5.210072492817937, "avg_headway_avs_mean": 7.751753132892433, "avg_headway_avs_min": 7.323345237995492, "avg_headway_avs_max": 8.196379075777283, "std_headway_avs_mean": 7.904581714941624, "std_headway_avs_min": 7.286151490944516, "std_headway_avs_max": 8.415161995678456, "avg_accel_avs_mean": 0.5988331723579536, "avg_accel_avs_min": 0.5643212178804713, "avg_accel_avs_max": 0.6171499444476003, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 21.5343308165635, "avg_mpg_per_veh_min": 20.913283623020863, "avg_mpg_per_veh_max": 22.5283478406038, "num_cars_mean": 898.7866, "num_cars_min": 891.861, "num_cars_max": 908.074, "avg_mpg_mean": 21.479107423692525, "avg_mpg_min": 20.873835366909578, "avg_mpg_max": 22.450157933151523}, "sampler_perf": {"mean_env_wait_ms": 1563.752178527444, "mean_processing_ms": 424.1515927898851, "mean_inference_ms": 1.8817013607020638}, "off_policy_estimator": {}, "info": {"num_steps_trained": 64493440, "num_steps_sampled": 935000, "sample_time_ms": 1987278.345, "load_time_ms": 486.664, "grad_time_ms": 187920.283, "update_time_ms": 7.69, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3840.8740234375, "policy_loss": 0.0001417632302036509, "vf_loss": 3840.8740234375, "vf_explained_var": 0.31076905131340027, "vf_preds": 126.0552978515625, "kl": 0.003955784719437361, "entropy": 0.705326497554779, "entropy_coeff": 0.0, "advantages": 2.350052909605438e-06, "rewards": 1.7803276777267456}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 935000, "episodes_total": 935, "training_iteration": 55, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_10-55-40", "timestamp": 1599130540, "time_this_iter_s": 2175.467256784439, "time_total_s": 121151.71959280968, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5f6d90>", "on_episode_step": ".on_episode_step at 0x7f3afc5f6e18>", "on_episode_end": ".on_episode_end at 0x7f3afc5f6ea0>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5f6f28>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 121151.71959280968, "timesteps_since_restore": 935000, "iterations_since_restore": 55, "perf": {"cpu_util_percent": 87.95182316876412, "ram_util_percent": 30.347370119393354}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 131959.38507484164, "episode_reward_min": 128325.92630306997, "episode_reward_mean": 130086.60553806603, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 589.9095307561965}, "policy_reward_mean": {"av": 411.9402309701574}, "custom_metrics": {"avg_speed_mean": 5.158836308250204, "avg_speed_min": 5.09923953951576, "avg_speed_max": 5.212584993868271, "avg_speed_avs_mean": 5.153811306228147, "avg_speed_avs_min": 5.095013043611431, "avg_speed_avs_max": 5.210072492817937, "avg_headway_avs_mean": 7.652860704457547, "avg_headway_avs_min": 7.115384289005528, "avg_headway_avs_max": 8.139705338084163, "std_headway_avs_mean": 7.889120232699192, "std_headway_avs_min": 7.286151490944516, "std_headway_avs_max": 8.415161995678456, "avg_accel_avs_mean": 0.5921273479384511, "avg_accel_avs_min": 0.5593436104503822, "avg_accel_avs_max": 0.6171499444476003, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 21.648808609884572, "avg_mpg_per_veh_min": 20.913283623020863, "avg_mpg_per_veh_max": 22.5283478406038, "num_cars_mean": 899.4639999999999, "num_cars_min": 891.861, "num_cars_max": 908.074, "avg_mpg_mean": 21.590839482015294, "avg_mpg_min": 20.873835366909578, "avg_mpg_max": 22.450157933151523}, "sampler_perf": {"mean_env_wait_ms": 1562.6933595351643, "mean_processing_ms": 424.6243095949609, "mean_inference_ms": 1.8800035189063828}, "off_policy_estimator": {}, "info": {"num_steps_trained": 65699456, "num_steps_sampled": 952000, "sample_time_ms": 1983802.725, "load_time_ms": 487.916, "grad_time_ms": 187551.682, "update_time_ms": 7.685, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3603.721923828125, "policy_loss": 0.00014856488269288093, "vf_loss": 3603.721435546875, "vf_explained_var": 0.35723966360092163, "vf_preds": 127.90593719482422, "kl": 0.0042162565514445305, "entropy": 0.6580178737640381, "entropy_coeff": 0.0, "advantages": 2.59613284470106e-06, "rewards": 1.7776941061019897}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 952000, "episodes_total": 952, "training_iteration": 56, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_11-31-35", "timestamp": 1599132695, "time_this_iter_s": 2155.000941991806, "time_total_s": 123306.72053480148, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc71e8c8>", "on_episode_step": ".on_episode_step at 0x7f3afc71ed90>", "on_episode_end": ".on_episode_end at 0x7f3afc75f268>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc75f158>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 123306.72053480148, "timesteps_since_restore": 952000, "iterations_since_restore": 56, "perf": {"cpu_util_percent": 87.27332247557003, "ram_util_percent": 30.356221498371337}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 132705.69228378226, "episode_reward_min": 128325.92630306997, "episode_reward_mean": 130514.83917157007, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 591.8285334322659}, "policy_reward_mean": {"av": 413.1000796719948}, "custom_metrics": {"avg_speed_mean": 5.153174556978829, "avg_speed_min": 5.09923953951576, "avg_speed_max": 5.212584993868271, "avg_speed_avs_mean": 5.148270414837209, "avg_speed_avs_min": 5.095013043611431, "avg_speed_avs_max": 5.208684270277558, "avg_headway_avs_mean": 7.6007080434098055, "avg_headway_avs_min": 7.115384289005528, "avg_headway_avs_max": 8.139705338084163, "std_headway_avs_mean": 7.872010918602505, "std_headway_avs_min": 7.286151490944516, "std_headway_avs_max": 8.415161995678456, "avg_accel_avs_mean": 0.5835435875348809, "avg_accel_avs_min": 0.5532274971253194, "avg_accel_avs_max": 0.6139305846276821, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 21.86076221046353, "avg_mpg_per_veh_min": 20.930143494501934, "avg_mpg_per_veh_max": 22.975243278464806, "num_cars_mean": 900.5180899999999, "num_cars_min": 891.861, "num_cars_max": 908.074, "avg_mpg_mean": 21.799882357705968, "avg_mpg_min": 20.894845422681534, "avg_mpg_max": 22.908937506082935}, "sampler_perf": {"mean_env_wait_ms": 1561.6952881285833, "mean_processing_ms": 425.0801829879929, "mean_inference_ms": 1.8783548668034995}, "off_policy_estimator": {}, "info": {"num_steps_trained": 66909696, "num_steps_sampled": 969000, "sample_time_ms": 1985341.899, "load_time_ms": 497.907, "grad_time_ms": 187837.308, "update_time_ms": 7.512, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3753.3359375, "policy_loss": 0.00019539051572792232, "vf_loss": 3753.3359375, "vf_explained_var": 0.3383975625038147, "vf_preds": 126.01445007324219, "kl": 0.0038109286688268185, "entropy": 0.6501306295394897, "entropy_coeff": 0.0, "advantages": 2.750588919298025e-06, "rewards": 1.7893401384353638}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 969000, "episodes_total": 969, "training_iteration": 57, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_12-08-11", "timestamp": 1599134891, "time_this_iter_s": 2195.370241165161, "time_total_s": 125502.09077596664, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc627730>", "on_episode_step": ".on_episode_step at 0x7f3afc627158>", "on_episode_end": ".on_episode_end at 0x7f3afc5f9ea0>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5f96a8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 125502.09077596664, "timesteps_since_restore": 969000, "iterations_since_restore": 57, "perf": {"cpu_util_percent": 87.73368522072937, "ram_util_percent": 30.461644273832377}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 133512.48276997052, "episode_reward_min": 128325.92630306997, "episode_reward_mean": 131096.5215626723, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 592.6972508192024}, "policy_reward_mean": {"av": 414.6000049420374}, "custom_metrics": {"avg_speed_mean": 5.146247967830954, "avg_speed_min": 5.095131835799204, "avg_speed_max": 5.20794487458526, "avg_speed_avs_mean": 5.141420315803981, "avg_speed_avs_min": 5.0918285739835625, "avg_speed_avs_max": 5.1987920289509555, "avg_headway_avs_mean": 7.524238822011132, "avg_headway_avs_min": 7.115384289005528, "avg_headway_avs_max": 8.01970542291144, "std_headway_avs_mean": 7.833133433537631, "std_headway_avs_min": 7.286151490944516, "std_headway_avs_max": 8.346627920076578, "avg_accel_avs_mean": 0.5725912166699243, "avg_accel_avs_min": 0.5322125361837732, "avg_accel_avs_max": 0.6139305846276821, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 22.124055399936747, "avg_mpg_per_veh_min": 20.930143494501934, "avg_mpg_per_veh_max": 23.14741044835222, "num_cars_mean": 902.1269199999998, "num_cars_min": 892.007, "num_cars_max": 911.312, "avg_mpg_mean": 22.059141763225252, "avg_mpg_min": 20.894845422681534, "avg_mpg_max": 23.05691652696393}, "sampler_perf": {"mean_env_wait_ms": 1560.8435304920279, "mean_processing_ms": 425.5266860683579, "mean_inference_ms": 1.8768349579985653}, "off_policy_estimator": {}, "info": {"num_steps_trained": 68123392, "num_steps_sampled": 986000, "sample_time_ms": 1988548.979, "load_time_ms": 499.964, "grad_time_ms": 188365.097, "update_time_ms": 7.657, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3684.7685546875, "policy_loss": 0.00011688092490658164, "vf_loss": 3684.7685546875, "vf_explained_var": 0.35922953486442566, "vf_preds": 126.82424926757812, "kl": 0.004529776517301798, "entropy": 0.6126237511634827, "entropy_coeff": 0.0, "advantages": 2.7002438400813844e-06, "rewards": 1.7938767671585083}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 986000, "episodes_total": 986, "training_iteration": 58, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_12-44-50", "timestamp": 1599137090, "time_this_iter_s": 2199.2149612903595, "time_total_s": 127701.305737257, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5e9d90>", "on_episode_step": ".on_episode_step at 0x7f3afc5e9e18>", "on_episode_end": ".on_episode_end at 0x7f3afc5e9ea0>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5e9f28>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 127701.305737257, "timesteps_since_restore": 986000, "iterations_since_restore": 58, "perf": {"cpu_util_percent": 88.13817427385892, "ram_util_percent": 30.374784551548036}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 133895.95908688253, "episode_reward_min": 128933.60138950343, "episode_reward_mean": 131734.21279799638, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 594.8454347074852}, "policy_reward_mean": {"av": 416.3665501374769}, "custom_metrics": {"avg_speed_mean": 5.138756165820694, "avg_speed_min": 5.095131835799204, "avg_speed_max": 5.20794487458526, "avg_speed_avs_mean": 5.134287069346225, "avg_speed_avs_min": 5.0918285739835625, "avg_speed_avs_max": 5.197466453115293, "avg_headway_avs_mean": 7.48056353428394, "avg_headway_avs_min": 7.115384289005528, "avg_headway_avs_max": 8.01970542291144, "std_headway_avs_mean": 7.794681483038841, "std_headway_avs_min": 7.286151490944516, "std_headway_avs_max": 8.346627920076578, "avg_accel_avs_mean": 0.5607177046379489, "avg_accel_avs_min": 0.5247501584375377, "avg_accel_avs_max": 0.6032946649590406, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 22.42437986076835, "avg_mpg_per_veh_min": 21.222643816242933, "avg_mpg_per_veh_max": 23.48140682778088, "num_cars_mean": 903.64062, "num_cars_min": 892.007, "num_cars_max": 911.312, "avg_mpg_mean": 22.35497390030213, "avg_mpg_min": 21.182925917607353, "avg_mpg_max": 23.401985062861524}, "sampler_perf": {"mean_env_wait_ms": 1560.0300199052833, "mean_processing_ms": 425.9547038248285, "mean_inference_ms": 1.8753496049016691}, "off_policy_estimator": {}, "info": {"num_steps_trained": 69336704, "num_steps_sampled": 1003000, "sample_time_ms": 1986447.169, "load_time_ms": 497.824, "grad_time_ms": 188106.641, "update_time_ms": 7.707, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3775.30615234375, "policy_loss": 0.00021784691489301622, "vf_loss": 3775.30615234375, "vf_explained_var": 0.33973604440689087, "vf_preds": 128.34564208984375, "kl": 0.004051777068525553, "entropy": 0.5945684909820557, "entropy_coeff": 0.0, "advantages": -1.031373813020764e-05, "rewards": 1.8018702268600464}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1003000, "episodes_total": 1003, "training_iteration": 59, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_13-20-53", "timestamp": 1599139253, "time_this_iter_s": 2162.80220580101, "time_total_s": 129864.10794305801, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5e9400>", "on_episode_step": ".on_episode_step at 0x7f3afc5e9488>", "on_episode_end": ".on_episode_end at 0x7f3afc5e9510>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc672e18>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 129864.10794305801, "timesteps_since_restore": 1003000, "iterations_since_restore": 59, "perf": {"cpu_util_percent": 87.15672077922076, "ram_util_percent": 30.38834415584416}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 133895.95908688253, "episode_reward_min": 129825.77981093306, "episode_reward_mean": 132194.91972207394, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 594.8454347074852}, "policy_reward_mean": {"av": 417.4532469828966}, "custom_metrics": {"avg_speed_mean": 5.1357352345091805, "avg_speed_min": 5.083889725097484, "avg_speed_max": 5.183894103748189, "avg_speed_avs_mean": 5.131542448959498, "avg_speed_avs_min": 5.077313752945712, "avg_speed_avs_max": 5.175427696162835, "avg_headway_avs_mean": 7.434865696864207, "avg_headway_avs_min": 7.092822180442688, "avg_headway_avs_max": 7.850369512873911, "std_headway_avs_mean": 7.771047306480165, "std_headway_avs_min": 7.258952124106794, "std_headway_avs_max": 8.346627920076578, "avg_accel_avs_mean": 0.5505290862069294, "avg_accel_avs_min": 0.5247501584375377, "avg_accel_avs_max": 0.5858742508198257, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 22.65450525690609, "avg_mpg_per_veh_min": 21.756422722279677, "avg_mpg_per_veh_max": 23.48140682778088, "num_cars_mean": 904.4443899999997, "num_cars_min": 897.256, "num_cars_max": 911.312, "avg_mpg_mean": 22.58179828106841, "avg_mpg_min": 21.693194966487756, "avg_mpg_max": 23.401985062861524}, "sampler_perf": {"mean_env_wait_ms": 1559.2401389095678, "mean_processing_ms": 426.3689076274591, "mean_inference_ms": 1.8739005429245454}, "off_policy_estimator": {}, "info": {"num_steps_trained": 70551296, "num_steps_sampled": 1020000, "sample_time_ms": 1988002.536, "load_time_ms": 495.541, "grad_time_ms": 188024.175, "update_time_ms": 7.958, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3879.744140625, "policy_loss": 0.00017454403860028833, "vf_loss": 3879.74365234375, "vf_explained_var": 0.347482830286026, "vf_preds": 127.51168823242188, "kl": 0.004187877755612135, "entropy": 0.5958242416381836, "entropy_coeff": 0.0, "advantages": 9.155742191069294e-07, "rewards": 1.7991235256195068}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1020000, "episodes_total": 1020, "training_iteration": 60, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_13-57-14", "timestamp": 1599141434, "time_this_iter_s": 2181.190890789032, "time_total_s": 132045.29883384705, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc597048>", "on_episode_step": ".on_episode_step at 0x7f3afc627950>", "on_episode_end": ".on_episode_end at 0x7f3afc627510>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc627e18>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 132045.29883384705, "timesteps_since_restore": 1020000, "iterations_since_restore": 60, "perf": {"cpu_util_percent": 87.27081724581723, "ram_util_percent": 30.461969111969108}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 134775.8210342995, "episode_reward_min": 129968.77239446662, "episode_reward_mean": 132704.48926495126, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 597.2874566814588}, "policy_reward_mean": {"av": 418.8507693872147}, "custom_metrics": {"avg_speed_mean": 5.12988689571493, "avg_speed_min": 5.0812886528387535, "avg_speed_max": 5.179666640114811, "avg_speed_avs_mean": 5.125943600605334, "avg_speed_avs_min": 5.076298498810461, "avg_speed_avs_max": 5.173554914576041, "avg_headway_avs_mean": 7.412237539892346, "avg_headway_avs_min": 7.092822180442688, "avg_headway_avs_max": 7.8042131902322724, "std_headway_avs_mean": 7.728213950203207, "std_headway_avs_min": 7.258952124106794, "std_headway_avs_max": 8.281984133320774, "avg_accel_avs_mean": 0.5415976973160422, "avg_accel_avs_min": 0.5168157041568918, "avg_accel_avs_max": 0.5746376679213977, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 22.90960534125996, "avg_mpg_per_veh_min": 21.760617602377753, "avg_mpg_per_veh_max": 24.09497262878448, "num_cars_mean": 905.68627, "num_cars_min": 897.256, "num_cars_max": 912.431, "avg_mpg_mean": 22.834496466145623, "avg_mpg_min": 21.693194966487756, "avg_mpg_max": 23.98478345755672}, "sampler_perf": {"mean_env_wait_ms": 1558.4831211273524, "mean_processing_ms": 426.7719969672356, "mean_inference_ms": 1.8724478175297536}, "off_policy_estimator": {}, "info": {"num_steps_trained": 71768832, "num_steps_sampled": 1037000, "sample_time_ms": 1987763.728, "load_time_ms": 495.13, "grad_time_ms": 188376.79, "update_time_ms": 7.959, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3897.550048828125, "policy_loss": 0.00023694826813880354, "vf_loss": 3897.549560546875, "vf_explained_var": 0.3402831554412842, "vf_preds": 125.77042388916016, "kl": 0.003962615504860878, "entropy": 0.5836312770843506, "entropy_coeff": 0.0, "advantages": 1.3060661331110168e-05, "rewards": 1.8057456016540527}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1037000, "episodes_total": 1037, "training_iteration": 61, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_14-33-39", "timestamp": 1599143619, "time_this_iter_s": 2184.746309041977, "time_total_s": 134230.04514288902, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5e9620>", "on_episode_step": ".on_episode_step at 0x7f3afc5e9268>", "on_episode_end": ".on_episode_end at 0x7f3afc5e92f0>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5e9378>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 134230.04514288902, "timesteps_since_restore": 1037000, "iterations_since_restore": 61, "perf": {"cpu_util_percent": 86.926735218509, "ram_util_percent": 30.44669023136247}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 135222.08223245436, "episode_reward_min": 131394.5252954204, "episode_reward_mean": 133238.8017317451, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 597.2874566814588}, "policy_reward_mean": {"av": 420.4708461617807}, "custom_metrics": {"avg_speed_mean": 5.1239660958840565, "avg_speed_min": 5.046585880359773, "avg_speed_max": 5.16757281042844, "avg_speed_avs_mean": 5.120536131807143, "avg_speed_avs_min": 5.046379004552357, "avg_speed_avs_max": 5.16372895594233, "avg_headway_avs_mean": 7.399611280742689, "avg_headway_avs_min": 6.796153135922731, "avg_headway_avs_max": 7.8042131902322724, "std_headway_avs_mean": 7.705592427525954, "std_headway_avs_min": 6.971688997822512, "std_headway_avs_max": 8.281984133320774, "avg_accel_avs_mean": 0.5335535242337714, "avg_accel_avs_min": 0.5115415048818073, "avg_accel_avs_max": 0.5607119390875702, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 23.16561372069746, "avg_mpg_per_veh_min": 22.25758810687895, "avg_mpg_per_veh_max": 24.164241436129938, "num_cars_mean": 906.9224100000001, "num_cars_min": 900.971, "num_cars_max": 917.498, "avg_mpg_mean": 23.088515770022774, "avg_mpg_min": 22.19486145821446, "avg_mpg_max": 24.04227203061831}, "sampler_perf": {"mean_env_wait_ms": 1557.8534029580042, "mean_processing_ms": 427.1638235765581, "mean_inference_ms": 1.8711374167560848}, "off_policy_estimator": {}, "info": {"num_steps_trained": 72985856, "num_steps_sampled": 1054000, "sample_time_ms": 1991539.779, "load_time_ms": 498.32, "grad_time_ms": 188604.522, "update_time_ms": 8.176, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3778.870849609375, "policy_loss": 0.00018737207574304193, "vf_loss": 3778.870849609375, "vf_explained_var": 0.3656857907772064, "vf_preds": 127.40541076660156, "kl": 0.004010602831840515, "entropy": 0.5546470284461975, "entropy_coeff": 0.0, "advantages": 9.411509381607175e-06, "rewards": 1.8057281970977783}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1054000, "episodes_total": 1054, "training_iteration": 62, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_15-10-24", "timestamp": 1599145824, "time_this_iter_s": 2205.1997871398926, "time_total_s": 136435.24493002892, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc597598>", "on_episode_step": ".on_episode_step at 0x7f3afc597488>", "on_episode_end": ".on_episode_end at 0x7f3afc597158>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5972f0>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 136435.24493002892, "timesteps_since_restore": 1054000, "iterations_since_restore": 62, "perf": {"cpu_util_percent": 88.49783508436802, "ram_util_percent": 30.402769818529126}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 135222.08223245436, "episode_reward_min": 131847.49722327842, "episode_reward_mean": 133433.90263527635, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 597.2874566814588}, "policy_reward_mean": {"av": 420.9138596109784}, "custom_metrics": {"avg_speed_mean": 5.1212637002855335, "avg_speed_min": 5.046585880359773, "avg_speed_max": 5.165752737362607, "avg_speed_avs_mean": 5.1179550883636065, "avg_speed_avs_min": 5.046379004552357, "avg_speed_avs_max": 5.161479982111503, "avg_headway_avs_mean": 7.370389666634245, "avg_headway_avs_min": 6.796153135922731, "avg_headway_avs_max": 7.8042131902322724, "std_headway_avs_mean": 7.689630770374497, "std_headway_avs_min": 6.971688997822512, "std_headway_avs_max": 8.281984133320774, "avg_accel_avs_mean": 0.5286953418609306, "avg_accel_avs_min": 0.5115415048818073, "avg_accel_avs_max": 0.5533892280681216, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 23.268442899592152, "avg_mpg_per_veh_min": 22.487476837678397, "avg_mpg_per_veh_max": 24.164241436129938, "num_cars_mean": 907.3784199999999, "num_cars_min": 899.471, "num_cars_max": 917.498, "avg_mpg_mean": 23.189121770640966, "avg_mpg_min": 22.399725346598615, "avg_mpg_max": 24.04227203061831}, "sampler_perf": {"mean_env_wait_ms": 1557.3245379919808, "mean_processing_ms": 427.5456389156183, "mean_inference_ms": 1.8699474362311834}, "off_policy_estimator": {}, "info": {"num_steps_trained": 74201728, "num_steps_sampled": 1071000, "sample_time_ms": 1995997.066, "load_time_ms": 501.501, "grad_time_ms": 189213.673, "update_time_ms": 7.978, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3908.56298828125, "policy_loss": 0.00026553997304290533, "vf_loss": 3908.56298828125, "vf_explained_var": 0.3735802471637726, "vf_preds": 128.43763732910156, "kl": 0.003967788070440292, "entropy": 0.5492127537727356, "entropy_coeff": 0.0, "advantages": -1.0185533028561622e-05, "rewards": 1.7990384101867676}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1071000, "episodes_total": 1071, "training_iteration": 63, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_15-47-19", "timestamp": 1599148039, "time_this_iter_s": 2215.039009809494, "time_total_s": 138650.2839398384, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5a4e18>", "on_episode_step": ".on_episode_step at 0x7f3afc5a4ea0>", "on_episode_end": ".on_episode_end at 0x7f3afc5a4f28>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc570048>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 138650.2839398384, "timesteps_since_restore": 1071000, "iterations_since_restore": 63, "perf": {"cpu_util_percent": 89.29974643423138, "ram_util_percent": 30.372773375594296}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 135222.08223245436, "episode_reward_min": 132195.15841629356, "episode_reward_mean": 133504.74209910096, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 597.2874566814588}, "policy_reward_mean": {"av": 421.23033412980664}, "custom_metrics": {"avg_speed_mean": 5.121028326653846, "avg_speed_min": 5.046585880359773, "avg_speed_max": 5.165752737362607, "avg_speed_avs_mean": 5.118112912567169, "avg_speed_avs_min": 5.046379004552357, "avg_speed_avs_max": 5.161479982111503, "avg_headway_avs_mean": 7.3518760885280265, "avg_headway_avs_min": 6.796153135922731, "avg_headway_avs_max": 7.8042131902322724, "std_headway_avs_mean": 7.6915916501057096, "std_headway_avs_min": 6.971688997822512, "std_headway_avs_max": 8.281984133320774, "avg_accel_avs_mean": 0.5271087181346628, "avg_accel_avs_min": 0.5115415048818073, "avg_accel_avs_max": 0.5434726183700179, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 23.307794003903073, "avg_mpg_per_veh_min": 22.792532708849684, "avg_mpg_per_veh_max": 24.164241436129938, "num_cars_mean": 907.40038, "num_cars_min": 899.471, "num_cars_max": 917.498, "avg_mpg_mean": 23.226362357754724, "avg_mpg_min": 22.702821736602214, "avg_mpg_max": 24.04227203061831}, "sampler_perf": {"mean_env_wait_ms": 1556.8335238354587, "mean_processing_ms": 427.9146640986106, "mean_inference_ms": 1.8688270111449719}, "off_policy_estimator": {}, "info": {"num_steps_trained": 75416960, "num_steps_sampled": 1088000, "sample_time_ms": 1997521.423, "load_time_ms": 494.357, "grad_time_ms": 189518.031, "update_time_ms": 8.07, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3932.346435546875, "policy_loss": 0.0002208134246757254, "vf_loss": 3932.346435546875, "vf_explained_var": 0.3887211084365845, "vf_preds": 126.71487426757812, "kl": 0.0038930971641093493, "entropy": 0.5624037981033325, "entropy_coeff": 0.0, "advantages": 1.1255135177634656e-05, "rewards": 1.798087477684021}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1088000, "episodes_total": 1088, "training_iteration": 64, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_16-24-05", "timestamp": 1599150245, "time_this_iter_s": 2205.441977739334, "time_total_s": 140855.72591757774, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5a4620>", "on_episode_step": ".on_episode_step at 0x7f3afc5a4488>", "on_episode_end": ".on_episode_end at 0x7f3afc5f96a8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5f9950>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 140855.72591757774, "timesteps_since_restore": 1088000, "iterations_since_restore": 64, "perf": {"cpu_util_percent": 88.47389366443808, "ram_util_percent": 30.41238459089462}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 135222.08223245436, "episode_reward_min": 132241.45296882372, "episode_reward_mean": 133579.25192559045, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 597.2874566814588}, "policy_reward_mean": {"av": 421.4255353048882}, "custom_metrics": {"avg_speed_mean": 5.118419792682095, "avg_speed_min": 5.046585880359773, "avg_speed_max": 5.1672050777984415, "avg_speed_avs_mean": 5.115435986356889, "avg_speed_avs_min": 5.046379004552357, "avg_speed_avs_max": 5.161479982111503, "avg_headway_avs_mean": 7.323270128494366, "avg_headway_avs_min": 6.796153135922731, "avg_headway_avs_max": 7.705513488057957, "std_headway_avs_mean": 7.661690624560814, "std_headway_avs_min": 6.971688997822512, "std_headway_avs_max": 8.161677920354855, "avg_accel_avs_mean": 0.5262886982012907, "avg_accel_avs_min": 0.5115415048818073, "avg_accel_avs_max": 0.5434726183700179, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 23.32280368953627, "avg_mpg_per_veh_min": 22.7939145931796, "avg_mpg_per_veh_max": 24.164241436129938, "num_cars_mean": 907.9168400000001, "num_cars_min": 899.471, "num_cars_max": 917.498, "avg_mpg_mean": 23.24233460517756, "avg_mpg_min": 22.702821736602214, "avg_mpg_max": 24.04227203061831}, "sampler_perf": {"mean_env_wait_ms": 1556.4406346778883, "mean_processing_ms": 428.2756789963675, "mean_inference_ms": 1.8678376722143375}, "off_policy_estimator": {}, "info": {"num_steps_trained": 76634240, "num_steps_sampled": 1105000, "sample_time_ms": 1999763.432, "load_time_ms": 493.071, "grad_time_ms": 189654.266, "update_time_ms": 8.408, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4035.907470703125, "policy_loss": 0.00023977157252375036, "vf_loss": 4035.907470703125, "vf_explained_var": 0.35996636748313904, "vf_preds": 125.69508361816406, "kl": 0.0037927012890577316, "entropy": 0.5801992416381836, "entropy_coeff": 0.0, "advantages": -3.7312288441171404e-06, "rewards": 1.8007028102874756}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1105000, "episodes_total": 1105, "training_iteration": 65, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_17-00-44", "timestamp": 1599152444, "time_this_iter_s": 2199.2427639961243, "time_total_s": 143054.96868157387, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc6c0158>", "on_episode_step": ".on_episode_step at 0x7f3afc6c0268>", "on_episode_end": ".on_episode_end at 0x7f3afc6c01e0>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc6c0048>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 143054.96868157387, "timesteps_since_restore": 1105000, "iterations_since_restore": 65, "perf": {"cpu_util_percent": 88.41848068943504, "ram_util_percent": 30.379891477816788}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 135222.08223245436, "episode_reward_min": 132241.45296882372, "episode_reward_mean": 133718.13030047034, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 597.2874566814588}, "policy_reward_mean": {"av": 422.06341234918983}, "custom_metrics": {"avg_speed_mean": 5.115953282581289, "avg_speed_min": 5.046585880359773, "avg_speed_max": 5.1672050777984415, "avg_speed_avs_mean": 5.1126791476774684, "avg_speed_avs_min": 5.046379004552357, "avg_speed_avs_max": 5.161479982111503, "avg_headway_avs_mean": 7.301962088810369, "avg_headway_avs_min": 6.796153135922731, "avg_headway_avs_max": 7.5989021085132915, "std_headway_avs_mean": 7.647501661450685, "std_headway_avs_min": 6.971688997822512, "std_headway_avs_max": 8.08067005800663, "avg_accel_avs_mean": 0.5243938891497126, "avg_accel_avs_min": 0.5115415048818073, "avg_accel_avs_max": 0.5400062270276494, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 23.394016239652082, "avg_mpg_per_veh_min": 22.7939145931796, "avg_mpg_per_veh_max": 24.164241436129938, "num_cars_mean": 908.4113699999999, "num_cars_min": 899.471, "num_cars_max": 917.498, "avg_mpg_mean": 23.31179221387028, "avg_mpg_min": 22.702821736602214, "avg_mpg_max": 24.04227203061831}, "sampler_perf": {"mean_env_wait_ms": 1556.0985230939957, "mean_processing_ms": 428.6257794067236, "mean_inference_ms": 1.8669220373996098}, "off_policy_estimator": {}, "info": {"num_steps_trained": 77853312, "num_steps_sampled": 1122000, "sample_time_ms": 2002649.885, "load_time_ms": 502.743, "grad_time_ms": 189394.282, "update_time_ms": 8.407, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3985.22265625, "policy_loss": 0.00018314100452698767, "vf_loss": 3985.22216796875, "vf_explained_var": 0.3550475537776947, "vf_preds": 124.32718658447266, "kl": 0.004054318647831678, "entropy": 0.5759039521217346, "entropy_coeff": 0.0, "advantages": 6.847093118267367e-06, "rewards": 1.8060797452926636}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1122000, "episodes_total": 1122, "training_iteration": 66, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_17-37-06", "timestamp": 1599154626, "time_this_iter_s": 2181.3755741119385, "time_total_s": 145236.3442556858, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc570b70>", "on_episode_step": ".on_episode_step at 0x7f3afc570ae8>", "on_episode_end": ".on_episode_end at 0x7f3afc570a60>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc570048>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 145236.3442556858, "timesteps_since_restore": 1122000, "iterations_since_restore": 66, "perf": {"cpu_util_percent": 87.73727067911167, "ram_util_percent": 30.43202446089476}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 135107.0776678789, "episode_reward_min": 132241.45296882372, "episode_reward_mean": 133692.7177556591, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 595.8478228538163}, "policy_reward_mean": {"av": 421.7302853400811}, "custom_metrics": {"avg_speed_mean": 5.11805971505633, "avg_speed_min": 5.0723273563860785, "avg_speed_max": 5.168646918603965, "avg_speed_avs_mean": 5.114607708123295, "avg_speed_avs_min": 5.069275514725985, "avg_speed_avs_max": 5.166119340291676, "avg_headway_avs_mean": 7.277967813859749, "avg_headway_avs_min": 6.909363261276001, "avg_headway_avs_max": 7.5989021085132915, "std_headway_avs_mean": 7.663921982214898, "std_headway_avs_min": 7.070906992382792, "std_headway_avs_max": 8.08067005800663, "avg_accel_avs_mean": 0.5232396773135979, "avg_accel_avs_min": 0.5052236601012406, "avg_accel_avs_max": 0.5400062270276494, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 23.382924366929863, "avg_mpg_per_veh_min": 22.7939145931796, "avg_mpg_per_veh_max": 23.96735886313139, "num_cars_mean": 908.1951999999999, "num_cars_min": 899.471, "num_cars_max": 916.082, "avg_mpg_mean": 23.299259867709903, "avg_mpg_min": 22.702821736602214, "avg_mpg_max": 23.888247850752396}, "sampler_perf": {"mean_env_wait_ms": 1555.765452774829, "mean_processing_ms": 428.9616931793534, "mean_inference_ms": 1.8660814560112677}, "off_policy_estimator": {}, "info": {"num_steps_trained": 79071360, "num_steps_sampled": 1139000, "sample_time_ms": 2001631.974, "load_time_ms": 498.288, "grad_time_ms": 189412.991, "update_time_ms": 8.353, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3758.352294921875, "policy_loss": 0.0002833460457623005, "vf_loss": 3758.351806640625, "vf_explained_var": 0.3758733868598938, "vf_preds": 124.44551086425781, "kl": 0.0037973227445036173, "entropy": 0.5210842490196228, "entropy_coeff": 0.0, "advantages": 5.3230501180223655e-06, "rewards": 1.8039580583572388}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1139000, "episodes_total": 1139, "training_iteration": 67, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_18-13-31", "timestamp": 1599156811, "time_this_iter_s": 2185.331286430359, "time_total_s": 147421.67554211617, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5e9d90>", "on_episode_step": ".on_episode_step at 0x7f3afc5e9e18>", "on_episode_end": ".on_episode_end at 0x7f3afc5e9268>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5e9378>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 147421.67554211617, "timesteps_since_restore": 1139000, "iterations_since_restore": 67, "perf": {"cpu_util_percent": 87.15833601027948, "ram_util_percent": 30.25788628332798}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 135107.0776678789, "episode_reward_min": 132241.45296882372, "episode_reward_mean": 133777.4401870187, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 595.8478228538163}, "policy_reward_mean": {"av": 421.85116103373707}, "custom_metrics": {"avg_speed_mean": 5.114025153273077, "avg_speed_min": 5.074843392513972, "avg_speed_max": 5.168646918603965, "avg_speed_avs_mean": 5.110469420593479, "avg_speed_avs_min": 5.070651289672944, "avg_speed_avs_max": 5.166119340291676, "avg_headway_avs_mean": 7.26397292867917, "avg_headway_avs_min": 6.909363261276001, "avg_headway_avs_max": 7.5989021085132915, "std_headway_avs_mean": 7.630362862316191, "std_headway_avs_min": 7.070906992382792, "std_headway_avs_max": 8.08067005800663, "avg_accel_avs_mean": 0.5210130816726866, "avg_accel_avs_min": 0.49893950217375077, "avg_accel_avs_max": 0.5400062270276494, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 23.443395736568622, "avg_mpg_per_veh_min": 22.7939145931796, "avg_mpg_per_veh_max": 24.146441389494182, "num_cars_mean": 908.5993500000001, "num_cars_min": 899.471, "num_cars_max": 916.082, "avg_mpg_mean": 23.356865247562567, "avg_mpg_min": 22.702821736602214, "avg_mpg_max": 24.001545588059564}, "sampler_perf": {"mean_env_wait_ms": 1555.4118442647589, "mean_processing_ms": 429.2879844166809, "mean_inference_ms": 1.865238060938851}, "off_policy_estimator": {}, "info": {"num_steps_trained": 80292352, "num_steps_sampled": 1156000, "sample_time_ms": 2000733.921, "load_time_ms": 495.674, "grad_time_ms": 189702.267, "update_time_ms": 8.535, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3720.404541015625, "policy_loss": 0.00028014686540700495, "vf_loss": 3720.404541015625, "vf_explained_var": 0.3696078956127167, "vf_preds": 121.71135711669922, "kl": 0.0038567185401916504, "entropy": 0.5106250047683716, "entropy_coeff": 0.0, "advantages": 9.276935998059344e-06, "rewards": 1.8047837018966675}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1156000, "episodes_total": 1156, "training_iteration": 68, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_18-50-04", "timestamp": 1599159004, "time_this_iter_s": 2193.0704798698425, "time_total_s": 149614.746021986, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc597268>", "on_episode_step": ".on_episode_step at 0x7f3afc597f28>", "on_episode_end": ".on_episode_end at 0x7f3afc597510>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc597620>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 149614.746021986, "timesteps_since_restore": 1156000, "iterations_since_restore": 68, "perf": {"cpu_util_percent": 87.67589628681178, "ram_util_percent": 30.348079385403327}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 135473.18663919988, "episode_reward_min": 132419.51601243642, "episode_reward_mean": 133984.7569197799, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 595.8478228538163}, "policy_reward_mean": {"av": 422.5182331676072}, "custom_metrics": {"avg_speed_mean": 5.112204483333813, "avg_speed_min": 5.055620960099241, "avg_speed_max": 5.168646918603965, "avg_speed_avs_mean": 5.108644297377182, "avg_speed_avs_min": 5.056388212474278, "avg_speed_avs_max": 5.166119340291676, "avg_headway_avs_mean": 7.2605991441674576, "avg_headway_avs_min": 6.885181785455375, "avg_headway_avs_max": 7.571707046495937, "std_headway_avs_mean": 7.6251097627467255, "std_headway_avs_min": 7.088844639430287, "std_headway_avs_max": 8.185279514888915, "avg_accel_avs_mean": 0.5165874575131898, "avg_accel_avs_min": 0.4914508635652361, "avg_accel_avs_max": 0.5400062270276494, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 23.559768376837496, "avg_mpg_per_veh_min": 22.7939145931796, "avg_mpg_per_veh_max": 24.233792953375197, "num_cars_mean": 909.20458, "num_cars_min": 899.964, "num_cars_max": 916.082, "avg_mpg_mean": 23.47030561272858, "avg_mpg_min": 22.702821736602214, "avg_mpg_max": 24.137528498874573}, "sampler_perf": {"mean_env_wait_ms": 1555.0242578650868, "mean_processing_ms": 429.60250772713204, "mean_inference_ms": 1.8643622646476516}, "off_policy_estimator": {}, "info": {"num_steps_trained": 81512576, "num_steps_sampled": 1173000, "sample_time_ms": 2004374.366, "load_time_ms": 494.158, "grad_time_ms": 190375.699, "update_time_ms": 8.515, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3640.19384765625, "policy_loss": 0.000210127531317994, "vf_loss": 3640.19384765625, "vf_explained_var": 0.38365593552589417, "vf_preds": 123.18730926513672, "kl": 0.004499634262174368, "entropy": 0.46873822808265686, "entropy_coeff": 0.0, "advantages": 2.9732295843132306e-06, "rewards": 1.8089078664779663}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1173000, "episodes_total": 1173, "training_iteration": 69, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_19-26-50", "timestamp": 1599161210, "time_this_iter_s": 2205.9885597229004, "time_total_s": 151820.7345817089, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5a4a60>", "on_episode_step": ".on_episode_step at 0x7f3afc5a4ae8>", "on_episode_end": ".on_episode_end at 0x7f3afc5a4c80>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5a4d08>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 151820.7345817089, "timesteps_since_restore": 1173000, "iterations_since_restore": 69, "perf": {"cpu_util_percent": 87.82867600254615, "ram_util_percent": 30.234627625716104}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 136882.45511178364, "episode_reward_min": 132775.08398826732, "episode_reward_mean": 134397.0172309908, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 604.1532850148697}, "policy_reward_mean": {"av": 423.56450435231903}, "custom_metrics": {"avg_speed_mean": 5.105373127588262, "avg_speed_min": 5.0493032446267065, "avg_speed_max": 5.168646918603965, "avg_speed_avs_mean": 5.101511666678293, "avg_speed_avs_min": 5.045034322653563, "avg_speed_avs_max": 5.166119340291676, "avg_headway_avs_mean": 7.258137426199936, "avg_headway_avs_min": 6.8827541536746395, "avg_headway_avs_max": 7.571707046495937, "std_headway_avs_mean": 7.5694548432910995, "std_headway_avs_min": 6.876174525246225, "std_headway_avs_max": 8.185279514888915, "avg_accel_avs_mean": 0.5096539419837636, "avg_accel_avs_min": 0.477099808072315, "avg_accel_avs_max": 0.5352925387363534, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 23.79416674341444, "avg_mpg_per_veh_min": 22.954989970264577, "avg_mpg_per_veh_max": 25.11004291494409, "num_cars_mean": 910.36938, "num_cars_min": 899.964, "num_cars_max": 922.133, "avg_mpg_mean": 23.703112428722065, "avg_mpg_min": 22.868251881120933, "avg_mpg_max": 25.017080088505644}, "sampler_perf": {"mean_env_wait_ms": 1554.6720340377535, "mean_processing_ms": 429.90938853691915, "mean_inference_ms": 1.8635343258420405}, "off_policy_estimator": {}, "info": {"num_steps_trained": 82739072, "num_steps_sampled": 1190000, "sample_time_ms": 2007257.016, "load_time_ms": 496.467, "grad_time_ms": 190748.991, "update_time_ms": 8.476, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3803.684814453125, "policy_loss": 0.0002836741623468697, "vf_loss": 3803.684814453125, "vf_explained_var": 0.35088831186294556, "vf_preds": 119.21715545654297, "kl": 0.004051356576383114, "entropy": 0.47489213943481445, "entropy_coeff": 0.0, "advantages": 3.2381085475208238e-06, "rewards": 1.8152656555175781}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1190000, "episodes_total": 1190, "training_iteration": 70, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_20-03-44", "timestamp": 1599163424, "time_this_iter_s": 2213.7479956150055, "time_total_s": 154034.4825773239, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc627400>", "on_episode_step": ".on_episode_step at 0x7f3afc627620>", "on_episode_end": ".on_episode_end at 0x7f3afc627378>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc627b70>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 154034.4825773239, "timesteps_since_restore": 1190000, "iterations_since_restore": 70, "perf": {"cpu_util_percent": 88.42635585156995, "ram_util_percent": 30.20662860767523}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 137404.03203585744, "episode_reward_min": 132930.48646447167, "episode_reward_mean": 134914.91592147297, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 606.5528181914538}, "policy_reward_mean": {"av": 424.94225305197966}, "custom_metrics": {"avg_speed_mean": 5.096001967432643, "avg_speed_min": 5.016740036972004, "avg_speed_max": 5.168646918603965, "avg_speed_avs_mean": 5.091955695440041, "avg_speed_avs_min": 5.014042554847776, "avg_speed_avs_max": 5.166119340291676, "avg_headway_avs_mean": 7.254273343127498, "avg_headway_avs_min": 6.8827541536746395, "avg_headway_avs_max": 7.571707046495937, "std_headway_avs_mean": 7.494004062351314, "std_headway_avs_min": 6.746797300195592, "std_headway_avs_max": 8.185279514888915, "avg_accel_avs_mean": 0.501193589152752, "avg_accel_avs_min": 0.4704922862828004, "avg_accel_avs_max": 0.5286175862457451, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 24.109002239160276, "avg_mpg_per_veh_min": 23.176909248706107, "avg_mpg_per_veh_max": 25.587502072914393, "num_cars_mean": 911.99593, "num_cars_min": 899.964, "num_cars_max": 925.296, "avg_mpg_mean": 24.013929898693597, "avg_mpg_min": 23.09977408655738, "avg_mpg_max": 25.48421447117545}, "sampler_perf": {"mean_env_wait_ms": 1554.3406842967656, "mean_processing_ms": 430.20956485756886, "mean_inference_ms": 1.8627771249232614}, "off_policy_estimator": {}, "info": {"num_steps_trained": 83971200, "num_steps_sampled": 1207000, "sample_time_ms": 2010147.095, "load_time_ms": 501.497, "grad_time_ms": 189995.58, "update_time_ms": 8.352, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3874.637451171875, "policy_loss": 0.000250514829531312, "vf_loss": 3874.636962890625, "vf_explained_var": 0.3026707172393799, "vf_preds": 114.01559448242188, "kl": 0.004069216549396515, "entropy": 0.5239632725715637, "entropy_coeff": 0.0, "advantages": 3.1977047001419123e-06, "rewards": 1.8218107223510742}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1207000, "episodes_total": 1207, "training_iteration": 71, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_20-40-30", "timestamp": 1599165630, "time_this_iter_s": 2206.133168697357, "time_total_s": 156240.61574602127, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc541840>", "on_episode_step": ".on_episode_step at 0x7f3afc5417b8>", "on_episode_end": ".on_episode_end at 0x7f3afc541730>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5416a8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 156240.61574602127, "timesteps_since_restore": 1207000, "iterations_since_restore": 71, "perf": {"cpu_util_percent": 87.51330362826225, "ram_util_percent": 30.216072565245064}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 137404.03203585744, "episode_reward_min": 133360.74561520718, "episode_reward_mean": 135226.31889152632, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 606.5528181914538}, "policy_reward_mean": {"av": 425.5477826463364}, "custom_metrics": {"avg_speed_mean": 5.090764232476647, "avg_speed_min": 5.016740036972004, "avg_speed_max": 5.168646918603965, "avg_speed_avs_mean": 5.087067772260905, "avg_speed_avs_min": 5.014042554847776, "avg_speed_avs_max": 5.166119340291676, "avg_headway_avs_mean": 7.233823461419057, "avg_headway_avs_min": 6.8827541536746395, "avg_headway_avs_max": 7.571707046495937, "std_headway_avs_mean": 7.448822668194713, "std_headway_avs_min": 6.746797300195592, "std_headway_avs_max": 8.185279514888915, "avg_accel_avs_mean": 0.49468459693507866, "avg_accel_avs_min": 0.4704922862828004, "avg_accel_avs_max": 0.5241017361618372, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 24.293704671984823, "avg_mpg_per_veh_min": 23.176909248706107, "avg_mpg_per_veh_max": 25.587502072914393, "num_cars_mean": 913.0162700000001, "num_cars_min": 899.964, "num_cars_max": 925.296, "avg_mpg_mean": 24.196578843807536, "avg_mpg_min": 23.09977408655738, "avg_mpg_max": 25.48421447117545}, "sampler_perf": {"mean_env_wait_ms": 1554.0583837830832, "mean_processing_ms": 430.4999247925354, "mean_inference_ms": 1.8621000511455406}, "off_policy_estimator": {}, "info": {"num_steps_trained": 85198720, "num_steps_sampled": 1224000, "sample_time_ms": 2010538.117, "load_time_ms": 502.345, "grad_time_ms": 190258.0, "update_time_ms": 8.396, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3733.694580078125, "policy_loss": 0.00030443776631727815, "vf_loss": 3733.6943359375, "vf_explained_var": 0.3425127863883972, "vf_preds": 116.78203582763672, "kl": 0.004118570126593113, "entropy": 0.4827438294887543, "entropy_coeff": 0.0, "advantages": -1.929518475662917e-06, "rewards": 1.8179675340652466}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1224000, "episodes_total": 1224, "training_iteration": 72, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_21-17-22", "timestamp": 1599167842, "time_this_iter_s": 2211.752456188202, "time_total_s": 158452.36820220947, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc672598>", "on_episode_step": ".on_episode_step at 0x7f3afc672620>", "on_episode_end": ".on_episode_end at 0x7f3afc672d90>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc627730>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 158452.36820220947, "timesteps_since_restore": 1224000, "iterations_since_restore": 72, "perf": {"cpu_util_percent": 87.7535702951444, "ram_util_percent": 30.206505871152014}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 137458.45249396501, "episode_reward_min": 133360.74561520718, "episode_reward_mean": 135669.3523634715, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 606.5528181914538}, "policy_reward_mean": {"av": 426.8748107843166}, "custom_metrics": {"avg_speed_mean": 5.085125170700085, "avg_speed_min": 5.016740036972004, "avg_speed_max": 5.137209454962104, "avg_speed_avs_mean": 5.08158608887964, "avg_speed_avs_min": 5.014042554847776, "avg_speed_avs_max": 5.133272442292347, "avg_headway_avs_mean": 7.205460788206465, "avg_headway_avs_min": 6.8827541536746395, "avg_headway_avs_max": 7.544597618562658, "std_headway_avs_mean": 7.390454983528279, "std_headway_avs_min": 6.746797300195592, "std_headway_avs_max": 8.185279514888915, "avg_accel_avs_mean": 0.4883636324577247, "avg_accel_avs_min": 0.4673612838370225, "avg_accel_avs_max": 0.5152932450073056, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 24.533818652502564, "avg_mpg_per_veh_min": 23.35950313939889, "avg_mpg_per_veh_max": 25.603894084835705, "num_cars_mean": 914.2759800000001, "num_cars_min": 906.617, "num_cars_max": 925.296, "avg_mpg_mean": 24.433467941728164, "avg_mpg_min": 23.274679404021494, "avg_mpg_max": 25.509023065737754}, "sampler_perf": {"mean_env_wait_ms": 1553.7957508471882, "mean_processing_ms": 430.7834409155525, "mean_inference_ms": 1.8614452121402736}, "off_policy_estimator": {}, "info": {"num_steps_trained": 86427904, "num_steps_sampled": 1241000, "sample_time_ms": 2007926.676, "load_time_ms": 502.621, "grad_time_ms": 190368.495, "update_time_ms": 8.555, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3760.439453125, "policy_loss": 0.00023670510563533753, "vf_loss": 3760.43896484375, "vf_explained_var": 0.35301777720451355, "vf_preds": 116.2305908203125, "kl": 0.003934833686798811, "entropy": 0.4920792877674103, "entropy_coeff": 0.0, "advantages": -5.613885605271207e-06, "rewards": 1.8234938383102417}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1241000, "episodes_total": 1241, "training_iteration": 73, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_21-53-52", "timestamp": 1599170032, "time_this_iter_s": 2190.0207154750824, "time_total_s": 160642.38891768456, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5f9ea0>", "on_episode_step": ".on_episode_step at 0x7f3afc5f9bf8>", "on_episode_end": ".on_episode_end at 0x7f3afc71e950>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc71e840>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 160642.38891768456, "timesteps_since_restore": 1241000, "iterations_since_restore": 73, "perf": {"cpu_util_percent": 86.76261622314844, "ram_util_percent": 30.274735492144917}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 137458.45249396501, "episode_reward_min": 133536.00544483808, "episode_reward_mean": 135980.25671035383, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 606.5528181914538}, "policy_reward_mean": {"av": 427.67811514500335}, "custom_metrics": {"avg_speed_mean": 5.082181780835255, "avg_speed_min": 5.016740036972004, "avg_speed_max": 5.137209454962104, "avg_speed_avs_mean": 5.07888134170945, "avg_speed_avs_min": 5.014042554847776, "avg_speed_avs_max": 5.133272442292347, "avg_headway_avs_mean": 7.167372255109037, "avg_headway_avs_min": 6.759126029238522, "avg_headway_avs_max": 7.544597618562658, "std_headway_avs_mean": 7.359060880521477, "std_headway_avs_min": 6.746797300195592, "std_headway_avs_max": 8.185279514888915, "avg_accel_avs_mean": 0.48313992759620755, "avg_accel_avs_min": 0.4673612838370225, "avg_accel_avs_max": 0.5063541750672055, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 24.70706376950017, "avg_mpg_per_veh_min": 23.35950313939889, "avg_mpg_per_veh_max": 25.603894084835705, "num_cars_mean": 915.22486, "num_cars_min": 906.617, "num_cars_max": 925.296, "avg_mpg_mean": 24.603542146075043, "avg_mpg_min": 23.274679404021494, "avg_mpg_max": 25.509023065737754}, "sampler_perf": {"mean_env_wait_ms": 1553.5129801575326, "mean_processing_ms": 431.0550265078067, "mean_inference_ms": 1.8607519160662225}, "off_policy_estimator": {}, "info": {"num_steps_trained": 87655680, "num_steps_sampled": 1258000, "sample_time_ms": 2006127.269, "load_time_ms": 507.214, "grad_time_ms": 189600.674, "update_time_ms": 8.624, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3621.632080078125, "policy_loss": 0.0003253960167057812, "vf_loss": 3621.632080078125, "vf_explained_var": 0.3825366497039795, "vf_preds": 115.9815444946289, "kl": 0.0039122034795582294, "entropy": 0.47597578167915344, "entropy_coeff": 0.0, "advantages": -5.185479949432192e-06, "rewards": 1.820099115371704}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1258000, "episodes_total": 1258, "training_iteration": 74, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_22-30-12", "timestamp": 1599172212, "time_this_iter_s": 2179.8412981033325, "time_total_s": 162822.2302157879, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc541ae8>", "on_episode_step": ".on_episode_step at 0x7f3afc5416a8>", "on_episode_end": ".on_episode_end at 0x7f3afc541c80>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc541bf8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 162822.2302157879, "timesteps_since_restore": 1258000, "iterations_since_restore": 74, "perf": {"cpu_util_percent": 86.55690821256039, "ram_util_percent": 30.278776167471822}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 137458.45249396501, "episode_reward_min": 134309.22286945363, "episode_reward_mean": 136265.6443650076, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 17.240176071797038}, "policy_reward_max": {"av": 606.5528181914538}, "policy_reward_mean": {"av": 428.2389829195714}, "custom_metrics": {"avg_speed_mean": 5.0800411897749855, "avg_speed_min": 5.016740036972004, "avg_speed_max": 5.137209454962104, "avg_speed_avs_mean": 5.076981071632322, "avg_speed_avs_min": 5.014042554847776, "avg_speed_avs_max": 5.133272442292347, "avg_headway_avs_mean": 7.160547299853659, "avg_headway_avs_min": 6.759126029238522, "avg_headway_avs_max": 7.512993670098363, "std_headway_avs_mean": 7.343501051125656, "std_headway_avs_min": 6.746797300195592, "std_headway_avs_max": 7.930928464271831, "avg_accel_avs_mean": 0.479265790369083, "avg_accel_avs_min": 0.4671296482257576, "avg_accel_avs_max": 0.49655440198941503, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 24.859366125253963, "avg_mpg_per_veh_min": 23.939727353845832, "avg_mpg_per_veh_max": 25.603894084835705, "num_cars_mean": 915.92335, "num_cars_min": 906.617, "num_cars_max": 925.296, "avg_mpg_mean": 24.755374319073336, "avg_mpg_min": 23.869184316550996, "avg_mpg_max": 25.509023065737754}, "sampler_perf": {"mean_env_wait_ms": 1553.2391599449913, "mean_processing_ms": 431.31856763134397, "mean_inference_ms": 1.8600797147349857}, "off_policy_estimator": {}, "info": {"num_steps_trained": 88881152, "num_steps_sampled": 1275000, "sample_time_ms": 2007335.969, "load_time_ms": 506.855, "grad_time_ms": 188270.328, "update_time_ms": 8.241, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3713.266845703125, "policy_loss": 0.00034502570633776486, "vf_loss": 3713.266357421875, "vf_explained_var": 0.38476911187171936, "vf_preds": 119.44586944580078, "kl": 0.003993012942373753, "entropy": 0.4691339433193207, "entropy_coeff": 0.0, "advantages": 6.505896408270928e-07, "rewards": 1.825468897819519}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1275000, "episodes_total": 1275, "training_iteration": 75, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_23-06-50", "timestamp": 1599174410, "time_this_iter_s": 2198.0178983211517, "time_total_s": 165020.24811410904, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5e9378>", "on_episode_step": ".on_episode_step at 0x7f3afc5e9d90>", "on_episode_end": ".on_episode_end at 0x7f3afc5e9e18>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5e9620>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 165020.24811410904, "timesteps_since_restore": 1275000, "iterations_since_restore": 75, "perf": {"cpu_util_percent": 86.73222612583838, "ram_util_percent": 30.405237943149153}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 138281.43385175706, "episode_reward_min": 134838.84342455454, "episode_reward_mean": 136566.84919730923, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 17.240176071797038}, "policy_reward_max": {"av": 609.6203588718614}, "policy_reward_mean": {"av": 429.2125501204011}, "custom_metrics": {"avg_speed_mean": 5.078541489289125, "avg_speed_min": 5.016740036972004, "avg_speed_max": 5.137209454962104, "avg_speed_avs_mean": 5.075559368890472, "avg_speed_avs_min": 5.014042554847776, "avg_speed_avs_max": 5.133272442292347, "avg_headway_avs_mean": 7.1346453712620015, "avg_headway_avs_min": 6.759126029238522, "avg_headway_avs_max": 7.512993670098363, "std_headway_avs_mean": 7.330770543397761, "std_headway_avs_min": 6.746797300195592, "std_headway_avs_max": 7.930928464271831, "avg_accel_avs_mean": 0.47518991400466315, "avg_accel_avs_min": 0.4567657955694866, "avg_accel_avs_max": 0.48933331850444256, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 25.03205935850097, "avg_mpg_per_veh_min": 24.280901364085032, "avg_mpg_per_veh_max": 25.91320220852144, "num_cars_mean": 916.49369, "num_cars_min": 906.617, "num_cars_max": 925.296, "avg_mpg_mean": 24.925992679948568, "avg_mpg_min": 24.187046475798102, "avg_mpg_max": 25.77922916231335}, "sampler_perf": {"mean_env_wait_ms": 1553.0098507582118, "mean_processing_ms": 431.56347395164806, "mean_inference_ms": 1.8594811561323525}, "off_policy_estimator": {}, "info": {"num_steps_trained": 90113408, "num_steps_sampled": 1292000, "sample_time_ms": 2010970.51, "load_time_ms": 499.83, "grad_time_ms": 188179.573, "update_time_ms": 8.247, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3891.0439453125, "policy_loss": 0.00024236888566520065, "vf_loss": 3891.0439453125, "vf_explained_var": 0.3609725832939148, "vf_preds": 118.49449157714844, "kl": 0.004139813128858805, "entropy": 0.4843122959136963, "entropy_coeff": 0.0, "advantages": -2.897230388043681e-06, "rewards": 1.8321971893310547}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1292000, "episodes_total": 1292, "training_iteration": 76, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-03_23-43-47", "timestamp": 1599176627, "time_this_iter_s": 2216.7905657291412, "time_total_s": 167237.03867983818, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc4fc950>", "on_episode_step": ".on_episode_step at 0x7f3afc4fc9d8>", "on_episode_end": ".on_episode_end at 0x7f3afc4fca60>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc4fcae8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 167237.03867983818, "timesteps_since_restore": 1292000, "iterations_since_restore": 76, "perf": {"cpu_util_percent": 86.82178594046866, "ram_util_percent": 30.35028499050032}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 138302.45250858075, "episode_reward_min": 134838.84342455454, "episode_reward_mean": 136671.12758335297, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 609.6203588718614}, "policy_reward_mean": {"av": 429.5672855901211}, "custom_metrics": {"avg_speed_mean": 5.082172752821389, "avg_speed_min": 5.032251446656753, "avg_speed_max": 5.126344965284599, "avg_speed_avs_mean": 5.079624306184045, "avg_speed_avs_min": 5.028312001944196, "avg_speed_avs_max": 5.125126431373045, "avg_headway_avs_mean": 7.136599684364153, "avg_headway_avs_min": 6.759126029238522, "avg_headway_avs_max": 7.592765204756211, "std_headway_avs_mean": 7.39541762603116, "std_headway_avs_min": 6.879646307703976, "std_headway_avs_max": 8.096786385507688, "avg_accel_avs_mean": 0.47166262134425857, "avg_accel_avs_min": 0.4476050803619807, "avg_accel_avs_max": 0.48933331850444256, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 25.10614121439536, "avg_mpg_per_veh_min": 24.280901364085032, "avg_mpg_per_veh_max": 26.083316676448412, "num_cars_mean": 916.2684100000001, "num_cars_min": 909.122, "num_cars_max": 924.773, "avg_mpg_mean": 24.99766006716901, "avg_mpg_min": 24.187046475798102, "avg_mpg_max": 26.001722812339153}, "sampler_perf": {"mean_env_wait_ms": 1552.802742117051, "mean_processing_ms": 431.7981263769169, "mean_inference_ms": 1.858855227237121}, "off_policy_estimator": {}, "info": {"num_steps_trained": 91341696, "num_steps_sampled": 1309000, "sample_time_ms": 2014144.908, "load_time_ms": 494.875, "grad_time_ms": 187205.589, "update_time_ms": 8.19, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3912.798583984375, "policy_loss": 0.0002557524712756276, "vf_loss": 3912.79833984375, "vf_explained_var": 0.38332903385162354, "vf_preds": 120.12420654296875, "kl": 0.004263098817318678, "entropy": 0.41876745223999023, "entropy_coeff": 0.0, "advantages": 8.547747825105034e-07, "rewards": 1.8354253768920898}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1309000, "episodes_total": 1309, "training_iteration": 77, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_00-20-34", "timestamp": 1599178834, "time_this_iter_s": 2207.326636314392, "time_total_s": 169444.36531615257, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afff9be18>", "on_episode_step": ".on_episode_step at 0x7f3afff9b158>", "on_episode_end": ".on_episode_end at 0x7f3afff9b510>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3b73421488>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 169444.36531615257, "timesteps_since_restore": 1309000, "iterations_since_restore": 77, "perf": {"cpu_util_percent": 87.07503180661578, "ram_util_percent": 30.314790076335882}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 138835.87192011066, "episode_reward_min": 135499.42446377408, "episode_reward_mean": 137030.99324990055, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 609.6203588718614}, "policy_reward_mean": {"av": 430.5630404383226}, "custom_metrics": {"avg_speed_mean": 5.079101367512664, "avg_speed_min": 5.01723140936781, "avg_speed_max": 5.126344965284599, "avg_speed_avs_mean": 5.076592832200716, "avg_speed_avs_min": 5.011999089113564, "avg_speed_avs_max": 5.125126431373045, "avg_headway_avs_mean": 7.121605614360944, "avg_headway_avs_min": 6.759126029238522, "avg_headway_avs_max": 7.592765204756211, "std_headway_avs_mean": 7.371109846466228, "std_headway_avs_min": 6.879646307703976, "std_headway_avs_max": 8.096786385507688, "avg_accel_avs_mean": 0.4664291949469172, "avg_accel_avs_min": 0.44373709439742426, "avg_accel_avs_max": 0.48668581977698727, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 25.316955521796366, "avg_mpg_per_veh_min": 24.39276122681794, "avg_mpg_per_veh_max": 26.342635964755008, "num_cars_mean": 917.08196, "num_cars_min": 909.122, "num_cars_max": 924.773, "avg_mpg_mean": 25.207076406105948, "avg_mpg_min": 24.294997061226677, "avg_mpg_max": 26.2294201530208}, "sampler_perf": {"mean_env_wait_ms": 1552.723809380071, "mean_processing_ms": 431.9976078931245, "mean_inference_ms": 1.8583262202466415}, "off_policy_estimator": {}, "info": {"num_steps_trained": 92576128, "num_steps_sampled": 1326000, "sample_time_ms": 2020085.12, "load_time_ms": 492.051, "grad_time_ms": 188364.372, "update_time_ms": 7.686, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4017.67236328125, "policy_loss": 0.0002795825421344489, "vf_loss": 4017.671875, "vf_explained_var": 0.3649135231971741, "vf_preds": 117.73343658447266, "kl": 0.003976380452513695, "entropy": 0.4366658627986908, "entropy_coeff": 0.0, "advantages": 2.5618253403081326e-06, "rewards": 1.8369083404541016}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1326000, "episodes_total": 1326, "training_iteration": 78, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_00-58-18", "timestamp": 1599181098, "time_this_iter_s": 2264.0518357753754, "time_total_s": 171708.41715192795, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5f99d8>", "on_episode_step": ".on_episode_step at 0x7f3afc5f9950>", "on_episode_end": ".on_episode_end at 0x7f3afc5f9d90>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5f9510>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 171708.41715192795, "timesteps_since_restore": 1326000, "iterations_since_restore": 78, "perf": {"cpu_util_percent": 85.83212403100775, "ram_util_percent": 30.376372093023257}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 139056.84595216208, "episode_reward_min": 135499.42446377408, "episode_reward_mean": 137335.42239755543, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 611.1123206842451}, "policy_reward_mean": {"av": 431.5467018525497}, "custom_metrics": {"avg_speed_mean": 5.075960893159002, "avg_speed_min": 5.01723140936781, "avg_speed_max": 5.126344965284599, "avg_speed_avs_mean": 5.073568990688863, "avg_speed_avs_min": 5.011999089113564, "avg_speed_avs_max": 5.125126431373045, "avg_headway_avs_mean": 7.144871541495132, "avg_headway_avs_min": 6.759126029238522, "avg_headway_avs_max": 7.592765204756211, "std_headway_avs_mean": 7.3680050369867205, "std_headway_avs_min": 6.879646307703976, "std_headway_avs_max": 8.096786385507688, "avg_accel_avs_mean": 0.4617165299815725, "avg_accel_avs_min": 0.4400434959788873, "avg_accel_avs_max": 0.484742910074552, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 25.516039240094905, "avg_mpg_per_veh_min": 24.39276122681794, "avg_mpg_per_veh_max": 26.543416630753637, "num_cars_mean": 917.9074800000001, "num_cars_min": 909.122, "num_cars_max": 927.816, "avg_mpg_mean": 25.404999620418586, "avg_mpg_min": 24.294997061226677, "avg_mpg_max": 26.41827345763349}, "sampler_perf": {"mean_env_wait_ms": 1552.855694244867, "mean_processing_ms": 432.1717012597659, "mean_inference_ms": 1.857999589202579}, "off_policy_estimator": {}, "info": {"num_steps_trained": 93809536, "num_steps_sampled": 1343000, "sample_time_ms": 2026449.616, "load_time_ms": 487.037, "grad_time_ms": 190478.726, "update_time_ms": 7.511, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4128.13427734375, "policy_loss": 0.0002585178008303046, "vf_loss": 4128.13427734375, "vf_explained_var": 0.34973201155662537, "vf_preds": 121.22662353515625, "kl": 0.004290998913347721, "entropy": 0.4476251006126404, "entropy_coeff": 0.0, "advantages": 6.399087055797281e-07, "rewards": 1.8407647609710693}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1343000, "episodes_total": 1343, "training_iteration": 79, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_01-36-29", "timestamp": 1599183389, "time_this_iter_s": 2290.666515111923, "time_total_s": 173999.08366703987, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc541bf8>", "on_episode_step": ".on_episode_step at 0x7f3afc541f28>", "on_episode_end": ".on_episode_end at 0x7f3afc541840>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc541ae8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 173999.08366703987, "timesteps_since_restore": 1343000, "iterations_since_restore": 79, "perf": {"cpu_util_percent": 86.79635193133046, "ram_util_percent": 30.39987737584304}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 139056.84595216208, "episode_reward_min": 135534.6690228553, "episode_reward_mean": 137651.18795789927, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 611.1123206842451}, "policy_reward_mean": {"av": 432.3487278029374}, "custom_metrics": {"avg_speed_mean": 5.07433116692366, "avg_speed_min": 5.01723140936781, "avg_speed_max": 5.126344965284599, "avg_speed_avs_mean": 5.071807280420486, "avg_speed_avs_min": 5.011999089113564, "avg_speed_avs_max": 5.1225595486120055, "avg_headway_avs_mean": 7.194559696299116, "avg_headway_avs_min": 6.790085304670585, "avg_headway_avs_max": 7.5941634758777745, "std_headway_avs_mean": 7.360567694678495, "std_headway_avs_min": 6.879646307703976, "std_headway_avs_max": 8.096786385507688, "avg_accel_avs_mean": 0.45669900809326797, "avg_accel_avs_min": 0.4394148507991186, "avg_accel_avs_max": 0.484742910074552, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 25.72209938842791, "avg_mpg_per_veh_min": 24.39276122681794, "avg_mpg_per_veh_max": 26.543416630753637, "num_cars_mean": 918.3944200000001, "num_cars_min": 909.122, "num_cars_max": 927.816, "avg_mpg_mean": 25.609111591357223, "avg_mpg_min": 24.294997061226677, "avg_mpg_max": 26.41827345763349}, "sampler_perf": {"mean_env_wait_ms": 1553.2302361516042, "mean_processing_ms": 432.2885832282913, "mean_inference_ms": 1.8579304071149598}, "off_policy_estimator": {}, "info": {"num_steps_trained": 95040000, "num_steps_sampled": 1360000, "sample_time_ms": 2032314.82, "load_time_ms": 484.136, "grad_time_ms": 192583.848, "update_time_ms": 7.072, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4154.337890625, "policy_loss": 0.0002669696696102619, "vf_loss": 4154.337890625, "vf_explained_var": 0.36135125160217285, "vf_preds": 122.90577697753906, "kl": 0.00458567263558507, "entropy": 0.3994404375553131, "entropy_coeff": 0.0, "advantages": -4.023846940981457e-06, "rewards": 1.8418395519256592}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1360000, "episodes_total": 1360, "training_iteration": 80, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_02-14-43", "timestamp": 1599185683, "time_this_iter_s": 2293.411235332489, "time_total_s": 176292.49490237236, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5e9620>", "on_episode_step": ".on_episode_step at 0x7f3afc4fcae8>", "on_episode_end": ".on_episode_end at 0x7f3afc4fca60>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc4fc9d8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 176292.49490237236, "timesteps_since_restore": 1360000, "iterations_since_restore": 80, "perf": {"cpu_util_percent": 86.54077134986224, "ram_util_percent": 30.388919498010406}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 139413.09613296707, "episode_reward_min": 136471.52958197767, "episode_reward_mean": 138033.65744942916, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 611.1123206842451}, "policy_reward_mean": {"av": 433.50917825894015}, "custom_metrics": {"avg_speed_mean": 5.069116538437447, "avg_speed_min": 5.01723140936781, "avg_speed_max": 5.121980166513662, "avg_speed_avs_mean": 5.066502541655577, "avg_speed_avs_min": 5.011999089113564, "avg_speed_avs_max": 5.1225595486120055, "avg_headway_avs_mean": 7.216549885567981, "avg_headway_avs_min": 6.790085304670585, "avg_headway_avs_max": 7.5941634758777745, "std_headway_avs_mean": 7.3214674721623725, "std_headway_avs_min": 6.879646307703976, "std_headway_avs_max": 8.096786385507688, "avg_accel_avs_mean": 0.45029371297132464, "avg_accel_avs_min": 0.4291301915188725, "avg_accel_avs_max": 0.4713367183868901, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 25.97886070315551, "avg_mpg_per_veh_min": 25.25287329043734, "avg_mpg_per_veh_max": 26.808812184566158, "num_cars_mean": 919.4288700000001, "num_cars_min": 910.237, "num_cars_max": 927.816, "avg_mpg_mean": 25.863235382414928, "avg_mpg_min": 25.144687047649196, "avg_mpg_max": 26.68085899017184}, "sampler_perf": {"mean_env_wait_ms": 1553.84693413706, "mean_processing_ms": 432.3438269422442, "mean_inference_ms": 1.8581089361659016}, "off_policy_estimator": {}, "info": {"num_steps_trained": 96274048, "num_steps_sampled": 1377000, "sample_time_ms": 2040532.89, "load_time_ms": 480.196, "grad_time_ms": 195059.399, "update_time_ms": 7.037, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4218.50048828125, "policy_loss": 0.00027763578691519797, "vf_loss": 4218.5009765625, "vf_explained_var": 0.3550291955471039, "vf_preds": 119.56732940673828, "kl": 0.004059921018779278, "entropy": 0.4067141115665436, "entropy_coeff": 0.0, "advantages": 1.7339195892418502e-06, "rewards": 1.8454381227493286}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1377000, "episodes_total": 1377, "training_iteration": 81, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_02-53-16", "timestamp": 1599187996, "time_this_iter_s": 2313.1023449897766, "time_total_s": 178605.59724736214, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc672730>", "on_episode_step": ".on_episode_step at 0x7f3afc672268>", "on_episode_end": ".on_episode_end at 0x7f3afc672400>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc672620>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 178605.59724736214, "timesteps_since_restore": 1377000, "iterations_since_restore": 81, "perf": {"cpu_util_percent": 87.45299119344062, "ram_util_percent": 30.397084725174615}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 139413.09613296707, "episode_reward_min": 136719.465333565, "episode_reward_mean": 138175.58217431325, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 611.1123206842451}, "policy_reward_mean": {"av": 433.8731502945749}, "custom_metrics": {"avg_speed_mean": 5.069964911662034, "avg_speed_min": 5.01723140936781, "avg_speed_max": 5.121980166513662, "avg_speed_avs_mean": 5.067393242096814, "avg_speed_avs_min": 5.011999089113564, "avg_speed_avs_max": 5.121040295033283, "avg_headway_avs_mean": 7.248685837045518, "avg_headway_avs_min": 6.8697508089084245, "avg_headway_avs_max": 7.5941634758777745, "std_headway_avs_mean": 7.335035827250676, "std_headway_avs_min": 6.691856000563164, "std_headway_avs_max": 8.096786385507688, "avg_accel_avs_mean": 0.4472922059032844, "avg_accel_avs_min": 0.4291301915188725, "avg_accel_avs_max": 0.4637405865896597, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 26.083804928175365, "avg_mpg_per_veh_min": 25.25287329043734, "avg_mpg_per_veh_max": 26.808812184566158, "num_cars_mean": 919.48262, "num_cars_min": 911.108, "num_cars_max": 927.816, "avg_mpg_mean": 25.966450528799623, "avg_mpg_min": 25.146582796019793, "avg_mpg_max": 26.68085899017184}, "sampler_perf": {"mean_env_wait_ms": 1554.6841333364316, "mean_processing_ms": 432.28550716192893, "mean_inference_ms": 1.8585557799201653}, "off_policy_estimator": {}, "info": {"num_steps_trained": 97505536, "num_steps_sampled": 1394000, "sample_time_ms": 2047825.408, "load_time_ms": 475.159, "grad_time_ms": 196830.337, "update_time_ms": 6.976, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4224.931640625, "policy_loss": 0.00029638063278980553, "vf_loss": 4224.93115234375, "vf_explained_var": 0.36184272170066833, "vf_preds": 120.28973388671875, "kl": 0.003998715430498123, "entropy": 0.4062580466270447, "entropy_coeff": 0.0, "advantages": -5.250053618510719e-07, "rewards": 1.8435922861099243}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1394000, "episodes_total": 1394, "training_iteration": 82, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_03-31-38", "timestamp": 1599190298, "time_this_iter_s": 2302.332488298416, "time_total_s": 180907.92973566055, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc672598>", "on_episode_step": ".on_episode_step at 0x7f3afc570ae8>", "on_episode_end": ".on_episode_end at 0x7f3afc5701e0>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc570158>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 180907.92973566055, "timesteps_since_restore": 1394000, "iterations_since_restore": 82, "perf": {"cpu_util_percent": 86.97346752058554, "ram_util_percent": 30.470844769746872}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 139675.03048062374, "episode_reward_min": 136719.465333565, "episode_reward_mean": 138419.6025268461, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 615.0094113076317}, "policy_reward_mean": {"av": 434.5029429225794}, "custom_metrics": {"avg_speed_mean": 5.065748442406179, "avg_speed_min": 5.016869707412068, "avg_speed_max": 5.1174450545667005, "avg_speed_avs_mean": 5.063099110105212, "avg_speed_avs_min": 5.011999089113564, "avg_speed_avs_max": 5.1144939686282544, "avg_headway_avs_mean": 7.261453908406037, "avg_headway_avs_min": 6.8697508089084245, "avg_headway_avs_max": 7.667612415174868, "std_headway_avs_mean": 7.284156036165406, "std_headway_avs_min": 6.691856000563164, "std_headway_avs_max": 7.9633091165317875, "avg_accel_avs_mean": 0.44391425513316696, "avg_accel_avs_min": 0.42744319185299845, "avg_accel_avs_max": 0.4606888229027989, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 26.248021573566653, "avg_mpg_per_veh_min": 25.61048340085189, "avg_mpg_per_veh_max": 26.905607417567644, "num_cars_mean": 920.4463199999998, "num_cars_min": 911.108, "num_cars_max": 929.763, "avg_mpg_mean": 26.128587710557085, "avg_mpg_min": 25.51131515953296, "avg_mpg_max": 26.802308220315922}, "sampler_perf": {"mean_env_wait_ms": 1555.757357507365, "mean_processing_ms": 432.12316778081396, "mean_inference_ms": 1.8592893725642412}, "off_policy_estimator": {}, "info": {"num_steps_trained": 98740992, "num_steps_sampled": 1411000, "sample_time_ms": 2059126.857, "load_time_ms": 474.247, "grad_time_ms": 198079.71, "update_time_ms": 7.045, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4301.32763671875, "policy_loss": 0.00025179149815812707, "vf_loss": 4301.3271484375, "vf_explained_var": 0.3346460163593292, "vf_preds": 116.72785186767578, "kl": 0.004277435131371021, "entropy": 0.414374440908432, "entropy_coeff": 0.0, "advantages": -4.832162971979415e-07, "rewards": 1.8459824323654175}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1411000, "episodes_total": 1411, "training_iteration": 83, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_04-10-14", "timestamp": 1599192614, "time_this_iter_s": 2315.543105840683, "time_total_s": 183223.47284150124, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc75ff28>", "on_episode_step": ".on_episode_step at 0x7f3afc75f158>", "on_episode_end": ".on_episode_end at 0x7f3afc75fe18>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc672d90>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 183223.47284150124, "timesteps_since_restore": 1411000, "iterations_since_restore": 83, "perf": {"cpu_util_percent": 87.95197089144935, "ram_util_percent": 30.4107337780473}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 139834.67895488683, "episode_reward_min": 136719.465333565, "episode_reward_mean": 138445.6353804159, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 615.0094113076317}, "policy_reward_mean": {"av": 434.5710194626652}, "custom_metrics": {"avg_speed_mean": 5.0680423660944, "avg_speed_min": 5.016869707412068, "avg_speed_max": 5.133370992766215, "avg_speed_avs_mean": 5.065598335193073, "avg_speed_avs_min": 5.013917839308311, "avg_speed_avs_max": 5.126921913764311, "avg_headway_avs_mean": 7.2680791155735935, "avg_headway_avs_min": 6.913954034006143, "avg_headway_avs_max": 7.762318240957953, "std_headway_avs_mean": 7.295657179454602, "std_headway_avs_min": 6.691856000563164, "std_headway_avs_max": 7.998027208496084, "avg_accel_avs_mean": 0.442168595691815, "avg_accel_avs_min": 0.42744319185299845, "avg_accel_avs_max": 0.4572071849582017, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 26.25916350766582, "avg_mpg_per_veh_min": 25.38135501363592, "avg_mpg_per_veh_max": 26.91746637033055, "num_cars_mean": 920.3058500000002, "num_cars_min": 910.53, "num_cars_max": 929.763, "avg_mpg_mean": 26.138915205111285, "avg_mpg_min": 25.27534490900335, "avg_mpg_max": 26.802308220315922}, "sampler_perf": {"mean_env_wait_ms": 1556.9687946029758, "mean_processing_ms": 431.8225479829609, "mean_inference_ms": 1.8602120625533294}, "off_policy_estimator": {}, "info": {"num_steps_trained": 99973248, "num_steps_sampled": 1428000, "sample_time_ms": 2066982.089, "load_time_ms": 466.072, "grad_time_ms": 199983.319, "update_time_ms": 7.258, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4139.97900390625, "policy_loss": 0.0004050332063343376, "vf_loss": 4139.97802734375, "vf_explained_var": 0.3757787346839905, "vf_preds": 117.94850158691406, "kl": 0.004096670541912317, "entropy": 0.3649248480796814, "entropy_coeff": 0.0, "advantages": 3.4667341424210463e-06, "rewards": 1.841884732246399}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1428000, "episodes_total": 1428, "training_iteration": 84, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_04-48-11", "timestamp": 1599194891, "time_this_iter_s": 2277.3864934444427, "time_total_s": 185500.85933494568, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc6c0b70>", "on_episode_step": ".on_episode_step at 0x7f3afc5700d0>", "on_episode_end": ".on_episode_end at 0x7f3afc541ae8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc541bf8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 185500.85933494568, "timesteps_since_restore": 1428000, "iterations_since_restore": 84, "perf": {"cpu_util_percent": 87.70893958076448, "ram_util_percent": 30.331165228113445}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 139834.67895488683, "episode_reward_min": 136719.465333565, "episode_reward_mean": 138434.92564534178, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 615.0094113076317}, "policy_reward_mean": {"av": 434.482849932025}, "custom_metrics": {"avg_speed_mean": 5.069075243783164, "avg_speed_min": 5.016869707412068, "avg_speed_max": 5.133370992766215, "avg_speed_avs_mean": 5.066816014454478, "avg_speed_avs_min": 5.017950252740898, "avg_speed_avs_max": 5.126921913764311, "avg_headway_avs_mean": 7.2626460599172775, "avg_headway_avs_min": 6.878280083205926, "avg_headway_avs_max": 7.762318240957953, "std_headway_avs_mean": 7.325972265545911, "std_headway_avs_min": 6.691856000563164, "std_headway_avs_max": 7.998027208496084, "avg_accel_avs_mean": 0.439970652017195, "avg_accel_avs_min": 0.42744319185299845, "avg_accel_avs_max": 0.45339607697339457, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 26.258841788496497, "avg_mpg_per_veh_min": 25.38135501363592, "avg_mpg_per_veh_max": 26.91746637033055, "num_cars_mean": 920.0660600000001, "num_cars_min": 910.53, "num_cars_max": 929.763, "avg_mpg_mean": 26.136139433598004, "avg_mpg_min": 25.27534490900335, "avg_mpg_max": 26.802308220315922}, "sampler_perf": {"mean_env_wait_ms": 1558.2605084125798, "mean_processing_ms": 431.3667974627848, "mean_inference_ms": 1.8612680307363099}, "off_policy_estimator": {}, "info": {"num_steps_trained": 101205632, "num_steps_sampled": 1445000, "sample_time_ms": 2070211.742, "load_time_ms": 464.506, "grad_time_ms": 201720.191, "update_time_ms": 7.355, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4158.1875, "policy_loss": 0.0003106897638645023, "vf_loss": 4158.18701171875, "vf_explained_var": 0.39396363496780396, "vf_preds": 118.15570831298828, "kl": 0.004186754114925861, "entropy": 0.3424323797225952, "entropy_coeff": 0.0, "advantages": 1.3301886383487727e-06, "rewards": 1.841354489326477}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1445000, "episodes_total": 1445, "training_iteration": 85, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_05-25-39", "timestamp": 1599197139, "time_this_iter_s": 2247.7303216457367, "time_total_s": 187748.58965659142, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afff9b510>", "on_episode_step": ".on_episode_step at 0x7f3afc6c0ae8>", "on_episode_end": ".on_episode_end at 0x7f3afc5a4488>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5a4ae8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 187748.58965659142, "timesteps_since_restore": 1445000, "iterations_since_restore": 85, "perf": {"cpu_util_percent": 88.5953154278576, "ram_util_percent": 30.386914428482196}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 139834.67895488683, "episode_reward_min": 137018.4272125537, "episode_reward_mean": 138419.70854326422, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 615.0094113076317}, "policy_reward_mean": {"av": 434.43509052559233}, "custom_metrics": {"avg_speed_mean": 5.069691169002258, "avg_speed_min": 5.016869707412068, "avg_speed_max": 5.133370992766215, "avg_speed_avs_mean": 5.067713089792364, "avg_speed_avs_min": 5.017950252740898, "avg_speed_avs_max": 5.126921913764311, "avg_headway_avs_mean": 7.2261941212278336, "avg_headway_avs_min": 6.816651052357982, "avg_headway_avs_max": 7.762318240957953, "std_headway_avs_mean": 7.338535314541428, "std_headway_avs_min": 6.691856000563164, "std_headway_avs_max": 7.998027208496084, "avg_accel_avs_mean": 0.44004109852951406, "avg_accel_avs_min": 0.42744319185299845, "avg_accel_avs_max": 0.45412162377268717, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 26.239433071102493, "avg_mpg_per_veh_min": 25.38135501363592, "avg_mpg_per_veh_max": 26.91746637033055, "num_cars_mean": 920.01224, "num_cars_min": 910.53, "num_cars_max": 929.763, "avg_mpg_mean": 26.11570520960008, "avg_mpg_min": 25.27534490900335, "avg_mpg_max": 26.802308220315922}, "sampler_perf": {"mean_env_wait_ms": 1559.5596505981746, "mean_processing_ms": 430.7606714677518, "mean_inference_ms": 1.862382692558169}, "off_policy_estimator": {}, "info": {"num_steps_trained": 102436608, "num_steps_sampled": 1462000, "sample_time_ms": 2069030.559, "load_time_ms": 457.207, "grad_time_ms": 202749.314, "update_time_ms": 7.293, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4099.9658203125, "policy_loss": 0.00023628631606698036, "vf_loss": 4099.9658203125, "vf_explained_var": 0.41176971793174744, "vf_preds": 117.12882232666016, "kl": 0.004317814484238625, "entropy": 0.35791563987731934, "entropy_coeff": 0.0, "advantages": -4.088782134203939e-06, "rewards": 1.840678095817566}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1462000, "episodes_total": 1462, "training_iteration": 86, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_06-02-34", "timestamp": 1599199354, "time_this_iter_s": 2215.1448822021484, "time_total_s": 189963.73453879356, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc704e18>", "on_episode_step": ".on_episode_step at 0x7f3afc75f9d8>", "on_episode_end": ".on_episode_end at 0x7f3afc75fe18>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc75f268>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 189963.73453879356, "timesteps_since_restore": 1462000, "iterations_since_restore": 86, "perf": {"cpu_util_percent": 86.70076069730587, "ram_util_percent": 30.498446909667198}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 139834.67895488683, "episode_reward_min": 137018.4272125537, "episode_reward_mean": 138343.10741297764, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 615.0094113076317}, "policy_reward_mean": {"av": 434.2083029816317}, "custom_metrics": {"avg_speed_mean": 5.072435873344779, "avg_speed_min": 5.016869707412068, "avg_speed_max": 5.134981627577338, "avg_speed_avs_mean": 5.070330988866457, "avg_speed_avs_min": 5.017950252740898, "avg_speed_avs_max": 5.135700515410957, "avg_headway_avs_mean": 7.199408962143297, "avg_headway_avs_min": 6.816651052357982, "avg_headway_avs_max": 7.762318240957953, "std_headway_avs_mean": 7.36679772952281, "std_headway_avs_min": 6.691856000563164, "std_headway_avs_max": 8.063053824540301, "avg_accel_avs_mean": 0.44162617799702936, "avg_accel_avs_min": 0.42744319185299845, "avg_accel_avs_max": 0.456547586008954, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 26.1720143700538, "avg_mpg_per_veh_min": 25.38135501363592, "avg_mpg_per_veh_max": 26.91746637033055, "num_cars_mean": 919.63663, "num_cars_min": 909.68, "num_cars_max": 929.763, "avg_mpg_mean": 26.048731330896018, "avg_mpg_min": 25.27534490900335, "avg_mpg_max": 26.802308220315922}, "sampler_perf": {"mean_env_wait_ms": 1560.845971489777, "mean_processing_ms": 430.0353224624746, "mean_inference_ms": 1.8634978459409646}, "off_policy_estimator": {}, "info": {"num_steps_trained": 103667840, "num_steps_sampled": 1479000, "sample_time_ms": 2070719.789, "load_time_ms": 460.551, "grad_time_ms": 204198.4, "update_time_ms": 7.554, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4104.75439453125, "policy_loss": 0.00024338050570804626, "vf_loss": 4104.75439453125, "vf_explained_var": 0.4101887345314026, "vf_preds": 115.84249877929688, "kl": 0.00437446404248476, "entropy": 0.37692326307296753, "entropy_coeff": 0.0, "advantages": -1.431233442872326e-07, "rewards": 1.8431199789047241}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1479000, "episodes_total": 1479, "training_iteration": 87, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_06-39-53", "timestamp": 1599201593, "time_this_iter_s": 2238.733682870865, "time_total_s": 192202.46822166443, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc541f28>", "on_episode_step": ".on_episode_step at 0x7f3afc541840>", "on_episode_end": ".on_episode_end at 0x7f3afc75f158>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc75ff28>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 192202.46822166443, "timesteps_since_restore": 1479000, "iterations_since_restore": 87, "perf": {"cpu_util_percent": 88.25098777046097, "ram_util_percent": 30.52063342740671}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 139961.84049035277, "episode_reward_min": 137018.4272125537, "episode_reward_mean": 138426.52990146144, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 616.9863728152275}, "policy_reward_mean": {"av": 434.5383284199568}, "custom_metrics": {"avg_speed_mean": 5.071774969829312, "avg_speed_min": 5.02321599571447, "avg_speed_max": 5.144759488416526, "avg_speed_avs_mean": 5.069772621284985, "avg_speed_avs_min": 5.01881295006844, "avg_speed_avs_max": 5.145086490742421, "avg_headway_avs_mean": 7.168087936412271, "avg_headway_avs_min": 6.795654011564202, "avg_headway_avs_max": 7.762318240957953, "std_headway_avs_mean": 7.361202062150433, "std_headway_avs_min": 6.764869473832649, "std_headway_avs_max": 8.109061903385184, "avg_accel_avs_mean": 0.4401802500062377, "avg_accel_avs_min": 0.4273849386788458, "avg_accel_avs_max": 0.456547586008954, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 26.2108151449875, "avg_mpg_per_veh_min": 25.38135501363592, "avg_mpg_per_veh_max": 26.91746637033055, "num_cars_mean": 919.9523400000002, "num_cars_min": 909.68, "num_cars_max": 927.692, "avg_mpg_mean": 26.08571674930233, "avg_mpg_min": 25.27534490900335, "avg_mpg_max": 26.802308220315922}, "sampler_perf": {"mean_env_wait_ms": 1562.074314911593, "mean_processing_ms": 429.2423135284031, "mean_inference_ms": 1.8645591941946358}, "off_policy_estimator": {}, "info": {"num_steps_trained": 104902272, "num_steps_sampled": 1496000, "sample_time_ms": 2068922.103, "load_time_ms": 462.304, "grad_time_ms": 203055.536, "update_time_ms": 8.039, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4041.293212890625, "policy_loss": 0.00022157968487590551, "vf_loss": 4041.292724609375, "vf_explained_var": 0.40797874331474304, "vf_preds": 113.0759506225586, "kl": 0.004417788702994585, "entropy": 0.3506762981414795, "entropy_coeff": 0.0, "advantages": -1.0654886182237533e-06, "rewards": 1.8467721939086914}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1496000, "episodes_total": 1496, "training_iteration": 88, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_07-17-08", "timestamp": 1599203828, "time_this_iter_s": 2234.661639690399, "time_total_s": 194437.12986135483, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc44c488>", "on_episode_step": ".on_episode_step at 0x7f3afc44c510>", "on_episode_end": ".on_episode_end at 0x7f3afc44c598>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc44c620>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 194437.12986135483, "timesteps_since_restore": 1496000, "iterations_since_restore": 88, "perf": {"cpu_util_percent": 88.08105560791707, "ram_util_percent": 30.362174049638707}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 139961.84049035277, "episode_reward_min": 137102.20475985168, "episode_reward_mean": 138418.87387737806, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 616.9863728152275}, "policy_reward_mean": {"av": 434.4461061403536}, "custom_metrics": {"avg_speed_mean": 5.0726044806670725, "avg_speed_min": 5.021685351754095, "avg_speed_max": 5.144759488416526, "avg_speed_avs_mean": 5.070654243697111, "avg_speed_avs_min": 5.0238900185986894, "avg_speed_avs_max": 5.145086490742421, "avg_headway_avs_mean": 7.143443976740115, "avg_headway_avs_min": 6.779539325432439, "avg_headway_avs_max": 7.762318240957953, "std_headway_avs_mean": 7.389029486942818, "std_headway_avs_min": 6.7807307768775935, "std_headway_avs_max": 8.109061903385184, "avg_accel_avs_mean": 0.43917356846456423, "avg_accel_avs_min": 0.4266725526947192, "avg_accel_avs_max": 0.456547586008954, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 26.207768409362963, "avg_mpg_per_veh_min": 25.38135501363592, "avg_mpg_per_veh_max": 26.91746637033055, "num_cars_mean": 919.8407500000001, "num_cars_min": 909.68, "num_cars_max": 927.236, "avg_mpg_mean": 26.08274540756922, "avg_mpg_min": 25.27534490900335, "avg_mpg_max": 26.78388127559305}, "sampler_perf": {"mean_env_wait_ms": 1563.189101863014, "mean_processing_ms": 428.45962166962994, "mean_inference_ms": 1.8655089994570762}, "off_policy_estimator": {}, "info": {"num_steps_trained": 106136704, "num_steps_sampled": 1513000, "sample_time_ms": 2065728.579, "load_time_ms": 463.201, "grad_time_ms": 201088.933, "update_time_ms": 8.167, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4093.4189453125, "policy_loss": 0.00022265022562351078, "vf_loss": 4093.41845703125, "vf_explained_var": 0.4008398652076721, "vf_preds": 111.53651428222656, "kl": 0.004350869916379452, "entropy": 0.3405147194862366, "entropy_coeff": 0.0, "advantages": 7.524969078076538e-06, "rewards": 1.845445156097412}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1513000, "episodes_total": 1513, "training_iteration": 89, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_07-54-27", "timestamp": 1599206067, "time_this_iter_s": 2239.092397212982, "time_total_s": 196676.2222585678, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc44cbf8>", "on_episode_step": ".on_episode_step at 0x7f3afc44cf28>", "on_episode_end": ".on_episode_end at 0x7f3afc44cea0>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc44ce18>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 196676.2222585678, "timesteps_since_restore": 1513000, "iterations_since_restore": 89, "perf": {"cpu_util_percent": 88.86688617121355, "ram_util_percent": 30.437692066478522}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 139961.84049035277, "episode_reward_min": 137102.20475985168, "episode_reward_mean": 138467.79713236488, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 616.9863728152275}, "policy_reward_mean": {"av": 434.7224574041345}, "custom_metrics": {"avg_speed_mean": 5.072820675786457, "avg_speed_min": 5.021685351754095, "avg_speed_max": 5.144759488416526, "avg_speed_avs_mean": 5.070707185384766, "avg_speed_avs_min": 5.0238900185986894, "avg_speed_avs_max": 5.145086490742421, "avg_headway_avs_mean": 7.1494988169918425, "avg_headway_avs_min": 6.779539325432439, "avg_headway_avs_max": 7.653659136451383, "std_headway_avs_mean": 7.3942133197402935, "std_headway_avs_min": 6.7807307768775935, "std_headway_avs_max": 8.109061903385184, "avg_accel_avs_mean": 0.4386638737526485, "avg_accel_avs_min": 0.4216427602492336, "avg_accel_avs_max": 0.456547586008954, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 26.254422601041625, "avg_mpg_per_veh_min": 25.512502125368297, "avg_mpg_per_veh_max": 27.140436752298015, "num_cars_mean": 919.8644400000001, "num_cars_min": 909.68, "num_cars_max": 927.236, "avg_mpg_mean": 26.127532148669967, "avg_mpg_min": 25.374889055114117, "avg_mpg_max": 26.98039062903269}, "sampler_perf": {"mean_env_wait_ms": 1564.1272951508197, "mean_processing_ms": 427.7454316744363, "mean_inference_ms": 1.8662701638869947}, "off_policy_estimator": {}, "info": {"num_steps_trained": 107369856, "num_steps_sampled": 1530000, "sample_time_ms": 2059631.737, "load_time_ms": 461.481, "grad_time_ms": 198830.489, "update_time_ms": 8.444, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4091.966796875, "policy_loss": 0.000284999783616513, "vf_loss": 4091.96630859375, "vf_explained_var": 0.3938908278942108, "vf_preds": 113.53958892822266, "kl": 0.00444670720025897, "entropy": 0.3384314775466919, "entropy_coeff": 0.0, "advantages": -1.8212862414657138e-05, "rewards": 1.8458983898162842}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1530000, "episodes_total": 1530, "training_iteration": 90, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_08-31-17", "timestamp": 1599208277, "time_this_iter_s": 2209.8279416561127, "time_total_s": 198886.05020022392, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc672598>", "on_episode_step": ".on_episode_step at 0x7f3afc672620>", "on_episode_end": ".on_episode_end at 0x7f3afc672400>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc672d90>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 198886.05020022392, "timesteps_since_restore": 1530000, "iterations_since_restore": 90, "perf": {"cpu_util_percent": 87.14196315120712, "ram_util_percent": 30.44961880559085}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 139961.84049035277, "episode_reward_min": 137092.32683666775, "episode_reward_mean": 138453.80525107548, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 616.9863728152275}, "policy_reward_mean": {"av": 434.71947392720483}, "custom_metrics": {"avg_speed_mean": 5.07288893825551, "avg_speed_min": 5.021685351754095, "avg_speed_max": 5.144759488416526, "avg_speed_avs_mean": 5.070602351769901, "avg_speed_avs_min": 5.0238900185986894, "avg_speed_avs_max": 5.145086490742421, "avg_headway_avs_mean": 7.154670220397237, "avg_headway_avs_min": 6.779539325432439, "avg_headway_avs_max": 7.653659136451383, "std_headway_avs_mean": 7.372858589826985, "std_headway_avs_min": 6.7807307768775935, "std_headway_avs_max": 8.109061903385184, "avg_accel_avs_mean": 0.4387484492059565, "avg_accel_avs_min": 0.4216427602492336, "avg_accel_avs_max": 0.456547586008954, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 26.260846662283647, "avg_mpg_per_veh_min": 25.566613325254796, "avg_mpg_per_veh_max": 27.140436752298015, "num_cars_mean": 919.9175699999998, "num_cars_min": 909.68, "num_cars_max": 927.236, "avg_mpg_mean": 26.134566617529423, "avg_mpg_min": 25.46016859909295, "avg_mpg_max": 26.98039062903269}, "sampler_perf": {"mean_env_wait_ms": 1564.9303165089473, "mean_processing_ms": 427.06951717301405, "mean_inference_ms": 1.8668902632761128}, "off_policy_estimator": {}, "info": {"num_steps_trained": 108601728, "num_steps_sampled": 1547000, "sample_time_ms": 2050233.421, "load_time_ms": 463.142, "grad_time_ms": 197301.043, "update_time_ms": 8.563, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4018.014892578125, "policy_loss": 0.0003462869208306074, "vf_loss": 4018.0146484375, "vf_explained_var": 0.40154537558555603, "vf_preds": 114.40721130371094, "kl": 0.004444826394319534, "entropy": 0.3178446590900421, "entropy_coeff": 0.0, "advantages": 2.64429490925977e-06, "rewards": 1.840396761894226}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1547000, "episodes_total": 1547, "training_iteration": 91, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_09-08-01", "timestamp": 1599210481, "time_this_iter_s": 2203.7371184825897, "time_total_s": 201089.7873187065, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc4fcf28>", "on_episode_step": ".on_episode_step at 0x7f3afc4fc598>", "on_episode_end": ".on_episode_end at 0x7f3afc4fc158>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc4fc950>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 201089.7873187065, "timesteps_since_restore": 1547000, "iterations_since_restore": 91, "perf": {"cpu_util_percent": 86.53045555909524, "ram_util_percent": 30.51586492513539}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 139961.84049035277, "episode_reward_min": 137092.32683666775, "episode_reward_mean": 138568.86630102745, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 616.9863728152275}, "policy_reward_mean": {"av": 435.16272430684114}, "custom_metrics": {"avg_speed_mean": 5.069977237728429, "avg_speed_min": 5.017710346835955, "avg_speed_max": 5.144759488416526, "avg_speed_avs_mean": 5.067855331659512, "avg_speed_avs_min": 5.0184818641023465, "avg_speed_avs_max": 5.145086490742421, "avg_headway_avs_mean": 7.184426519905037, "avg_headway_avs_min": 6.779539325432439, "avg_headway_avs_max": 7.708074673287138, "std_headway_avs_mean": 7.34023299895812, "std_headway_avs_min": 6.674599619895312, "std_headway_avs_max": 8.109061903385184, "avg_accel_avs_mean": 0.43682443475288735, "avg_accel_avs_min": 0.4216427602492336, "avg_accel_avs_max": 0.456547586008954, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 26.348824310682872, "avg_mpg_per_veh_min": 25.627129151884628, "avg_mpg_per_veh_max": 27.140436752298015, "num_cars_mean": 920.52291, "num_cars_min": 909.68, "num_cars_max": 927.67, "avg_mpg_mean": 26.22370267000851, "avg_mpg_min": 25.492925290274794, "avg_mpg_max": 26.98039062903269}, "sampler_perf": {"mean_env_wait_ms": 1565.6508516578535, "mean_processing_ms": 426.46114641246646, "mean_inference_ms": 1.8674097223793933}, "off_policy_estimator": {}, "info": {"num_steps_trained": 109835136, "num_steps_sampled": 1564000, "sample_time_ms": 2043866.628, "load_time_ms": 467.093, "grad_time_ms": 195248.889, "update_time_ms": 8.55, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4065.7529296875, "policy_loss": 0.0003071401733905077, "vf_loss": 4065.752685546875, "vf_explained_var": 0.37860342860221863, "vf_preds": 115.32911682128906, "kl": 0.004295310936868191, "entropy": 0.3431489169597626, "entropy_coeff": 0.0, "advantages": 1.1428028301452287e-06, "rewards": 1.8467456102371216}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1564000, "episodes_total": 1564, "training_iteration": 92, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_09-44-59", "timestamp": 1599212699, "time_this_iter_s": 2218.1793105602264, "time_total_s": 203307.96662926674, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5f9730>", "on_episode_step": ".on_episode_step at 0x7f3afc5f98c8>", "on_episode_end": ".on_episode_end at 0x7f3afc5f9bf8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5f99d8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 203307.96662926674, "timesteps_since_restore": 1564000, "iterations_since_restore": 92, "perf": {"cpu_util_percent": 86.94051915163026, "ram_util_percent": 30.625197847420065}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 140422.53687966804, "episode_reward_min": 137092.32683666775, "episode_reward_mean": 138823.13312954112, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 616.9863728152275}, "policy_reward_mean": {"av": 435.8106772447451}, "custom_metrics": {"avg_speed_mean": 5.063566134732982, "avg_speed_min": 5.013355729576144, "avg_speed_max": 5.144759488416526, "avg_speed_avs_mean": 5.061728850441254, "avg_speed_avs_min": 5.004063730965045, "avg_speed_avs_max": 5.145086490742421, "avg_headway_avs_mean": 7.201145083884897, "avg_headway_avs_min": 6.779539325432439, "avg_headway_avs_max": 7.708074673287138, "std_headway_avs_mean": 7.244867964391591, "std_headway_avs_min": 6.578462289774734, "std_headway_avs_max": 8.109061903385184, "avg_accel_avs_mean": 0.43222133070669494, "avg_accel_avs_min": 0.40943049031449025, "avg_accel_avs_max": 0.45118310634234654, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 26.54498666180444, "avg_mpg_per_veh_min": 25.627129151884628, "avg_mpg_per_veh_max": 27.546587537338628, "num_cars_mean": 921.71178, "num_cars_min": 911.562, "num_cars_max": 932.474, "avg_mpg_mean": 26.41856701507686, "avg_mpg_min": 25.492925290274794, "avg_mpg_max": 27.416169073393363}, "sampler_perf": {"mean_env_wait_ms": 1566.3048676750511, "mean_processing_ms": 425.920557560357, "mean_inference_ms": 1.8678599872381594}, "off_policy_estimator": {}, "info": {"num_steps_trained": 111075072, "num_steps_sampled": 1581000, "sample_time_ms": 2037776.977, "load_time_ms": 465.619, "grad_time_ms": 193854.326, "update_time_ms": 8.694, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4192.79052734375, "policy_loss": 0.0002660407335497439, "vf_loss": 4192.78955078125, "vf_explained_var": 0.3172534704208374, "vf_preds": 112.69768524169922, "kl": 0.004134379792958498, "entropy": 0.35462257266044617, "entropy_coeff": 0.0, "advantages": 1.608254791563013e-07, "rewards": 1.8519338369369507}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1581000, "episodes_total": 1581, "training_iteration": 93, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_10-22-20", "timestamp": 1599214940, "time_this_iter_s": 2240.643602371216, "time_total_s": 205548.61023163795, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc44cea0>", "on_episode_step": ".on_episode_step at 0x7f3afc44c730>", "on_episode_end": ".on_episode_end at 0x7f3afc44c048>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc44c0d0>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 205548.61023163795, "timesteps_since_restore": 1581000, "iterations_since_restore": 93, "perf": {"cpu_util_percent": 88.25357142857143, "ram_util_percent": 30.587249373433583}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 140976.39616670346, "episode_reward_min": 137092.32683666775, "episode_reward_mean": 139121.85787695358, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 612.0014218732617}, "policy_reward_mean": {"av": 436.5977024225751}, "custom_metrics": {"avg_speed_mean": 5.054680647316718, "avg_speed_min": 4.982228342022576, "avg_speed_max": 5.112367023989925, "avg_speed_avs_mean": 5.053072203385505, "avg_speed_avs_min": 4.977843973012474, "avg_speed_avs_max": 5.115349245409685, "avg_headway_avs_mean": 7.200863615755298, "avg_headway_avs_min": 6.779539325432439, "avg_headway_avs_max": 7.708074673287138, "std_headway_avs_mean": 7.123666954643027, "std_headway_avs_min": 6.236126949934599, "std_headway_avs_max": 7.985440754958648, "avg_accel_avs_mean": 0.42782756819463236, "avg_accel_avs_min": 0.4039917788406634, "avg_accel_avs_max": 0.45118310634234654, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 26.761830928536877, "avg_mpg_per_veh_min": 25.627129151884628, "avg_mpg_per_veh_max": 28.00392204096709, "num_cars_mean": 923.28287, "num_cars_min": 912.211, "num_cars_max": 935.086, "avg_mpg_mean": 26.636482956006567, "avg_mpg_min": 25.492925290274794, "avg_mpg_max": 27.866052640479943}, "sampler_perf": {"mean_env_wait_ms": 1566.9099250213424, "mean_processing_ms": 425.4735327080256, "mean_inference_ms": 1.8682258122164817}, "off_policy_estimator": {}, "info": {"num_steps_trained": 112320640, "num_steps_sampled": 1598000, "sample_time_ms": 2036452.599, "load_time_ms": 473.718, "grad_time_ms": 192035.245, "update_time_ms": 8.376, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4200.720703125, "policy_loss": 0.00035479184589348733, "vf_loss": 4200.7197265625, "vf_explained_var": 0.282176673412323, "vf_preds": 107.57571411132812, "kl": 0.004189022351056337, "entropy": 0.371834397315979, "entropy_coeff": 0.0, "advantages": -1.8880465404436109e-06, "rewards": 1.8518909215927124}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1598000, "episodes_total": 1598, "training_iteration": 94, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_10-59-46", "timestamp": 1599217186, "time_this_iter_s": 2246.002203941345, "time_total_s": 207794.6124355793, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc3b0730>", "on_episode_step": ".on_episode_step at 0x7f3afc3b07b8>", "on_episode_end": ".on_episode_end at 0x7f3afc3b0840>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc3b08c8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 207794.6124355793, "timesteps_since_restore": 1598000, "iterations_since_restore": 94, "perf": {"cpu_util_percent": 89.23188496405126, "ram_util_percent": 30.506877149109098}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 141420.81768915342, "episode_reward_min": 137092.32683666775, "episode_reward_mean": 139446.03512383695, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 612.2453713504558}, "policy_reward_mean": {"av": 437.3954239949715}, "custom_metrics": {"avg_speed_mean": 5.044774396326742, "avg_speed_min": 4.9696759717720145, "avg_speed_max": 5.112367023989925, "avg_speed_avs_mean": 5.043181184729003, "avg_speed_avs_min": 4.970744172733791, "avg_speed_avs_max": 5.115349245409685, "avg_headway_avs_mean": 7.2082651626064616, "avg_headway_avs_min": 6.804819740281156, "avg_headway_avs_max": 7.708074673287138, "std_headway_avs_mean": 6.992237627104137, "std_headway_avs_min": 6.219801379515592, "std_headway_avs_max": 7.694062718927412, "avg_accel_avs_mean": 0.4235249348215927, "avg_accel_avs_min": 0.40250319796057565, "avg_accel_avs_max": 0.45118310634234654, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 26.99779883919483, "avg_mpg_per_veh_min": 25.627129151884628, "avg_mpg_per_veh_max": 28.252561312176667, "num_cars_mean": 925.1693799999999, "num_cars_min": 912.533, "num_cars_max": 937.786, "avg_mpg_mean": 26.871799953590084, "avg_mpg_min": 25.492925290274794, "avg_mpg_max": 28.130547600177053}, "sampler_perf": {"mean_env_wait_ms": 1567.4927936806455, "mean_processing_ms": 425.0658301521053, "mean_inference_ms": 1.868552819694893}, "off_policy_estimator": {}, "info": {"num_steps_trained": 113566848, "num_steps_sampled": 1615000, "sample_time_ms": 2036202.358, "load_time_ms": 473.795, "grad_time_ms": 190892.692, "update_time_ms": 8.395, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4335.85498046875, "policy_loss": 0.00030347605934366584, "vf_loss": 4335.8544921875, "vf_explained_var": 0.24462108314037323, "vf_preds": 107.67037963867188, "kl": 0.004096957389265299, "entropy": 0.38295093178749084, "entropy_coeff": 0.0, "advantages": -7.308648491743952e-06, "rewards": 1.85300612449646}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1615000, "episodes_total": 1615, "training_iteration": 95, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_11-36-59", "timestamp": 1599219419, "time_this_iter_s": 2233.765289068222, "time_total_s": 210028.37772464752, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc570048>", "on_episode_step": ".on_episode_step at 0x7f3afc570ae8>", "on_episode_end": ".on_episode_end at 0x7f3afc570bf8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc570b70>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 210028.37772464752, "timesteps_since_restore": 1615000, "iterations_since_restore": 95, "perf": {"cpu_util_percent": 88.78170961659335, "ram_util_percent": 30.421684475172842}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 141477.36294169014, "episode_reward_min": 137229.4242886712, "episode_reward_mean": 139827.20505730913, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 615.3033606801852}, "policy_reward_mean": {"av": 438.3572796329208}, "custom_metrics": {"avg_speed_mean": 5.030722857210673, "avg_speed_min": 4.945469786063482, "avg_speed_max": 5.112367023989925, "avg_speed_avs_mean": 5.0291139357768255, "avg_speed_avs_min": 4.944639934955245, "avg_speed_avs_max": 5.115349245409685, "avg_headway_avs_mean": 7.215486722415043, "avg_headway_avs_min": 6.804819740281156, "avg_headway_avs_max": 7.708074673287138, "std_headway_avs_mean": 6.845550871521121, "std_headway_avs_min": 6.083222494490728, "std_headway_avs_max": 7.694062718927412, "avg_accel_avs_mean": 0.41720440393228564, "avg_accel_avs_min": 0.39110864395883016, "avg_accel_avs_max": 0.44506549361068987, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 27.276307707330208, "avg_mpg_per_veh_min": 25.71116926147662, "avg_mpg_per_veh_max": 28.41661550534895, "num_cars_mean": 927.7915600000001, "num_cars_min": 914.678, "num_cars_max": 941.233, "avg_mpg_mean": 27.152663226356285, "avg_mpg_min": 25.596843375152208, "avg_mpg_max": 28.29127023288229}, "sampler_perf": {"mean_env_wait_ms": 1568.117276014835, "mean_processing_ms": 424.69845390683105, "mean_inference_ms": 1.8688838964842325}, "off_policy_estimator": {}, "info": {"num_steps_trained": 114816128, "num_steps_sampled": 1632000, "sample_time_ms": 2039656.135, "load_time_ms": 474.603, "grad_time_ms": 190185.923, "update_time_ms": 8.705, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4379.8291015625, "policy_loss": 0.0002807097334880382, "vf_loss": 4379.8291015625, "vf_explained_var": 0.2066650688648224, "vf_preds": 107.39342498779297, "kl": 0.004042602609843016, "entropy": 0.38556063175201416, "entropy_coeff": 0.0, "advantages": 9.67631990533846e-07, "rewards": 1.8493680953979492}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1632000, "episodes_total": 1632, "training_iteration": 96, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_12-14-22", "timestamp": 1599221662, "time_this_iter_s": 2242.6695425510406, "time_total_s": 212271.04726719856, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc4fc7b8>", "on_episode_step": ".on_episode_step at 0x7f3afc4fc840>", "on_episode_end": ".on_episode_end at 0x7f3afc4fc488>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc4fc268>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 212271.04726719856, "timesteps_since_restore": 1632000, "iterations_since_restore": 96, "perf": {"cpu_util_percent": 88.67961803381338, "ram_util_percent": 30.45122103944897}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 141477.36294169014, "episode_reward_min": 137928.46411015902, "episode_reward_mean": 140172.05604673031, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 615.3033606801852}, "policy_reward_mean": {"av": 439.5761918173929}, "custom_metrics": {"avg_speed_mean": 5.0138431623869595, "avg_speed_min": 4.940294961190172, "avg_speed_max": 5.1037904488321475, "avg_speed_avs_mean": 5.011827058620239, "avg_speed_avs_min": 4.938216188844892, "avg_speed_avs_max": 5.1032001542680465, "avg_headway_avs_mean": 7.257930868900176, "avg_headway_avs_min": 6.804819740281156, "avg_headway_avs_max": 7.778997806489627, "std_headway_avs_mean": 6.718229945495481, "std_headway_avs_min": 6.083222494490728, "std_headway_avs_max": 7.641209558922362, "avg_accel_avs_mean": 0.41161296481188553, "avg_accel_avs_min": 0.39110864395883016, "avg_accel_avs_max": 0.44405929582728787, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 27.576423842659132, "avg_mpg_per_veh_min": 26.145671915605163, "avg_mpg_per_veh_max": 28.41661550534895, "num_cars_mean": 930.7256399999997, "num_cars_min": 914.804, "num_cars_max": 942.886, "avg_mpg_mean": 27.453098767388415, "avg_mpg_min": 26.020340231390314, "avg_mpg_max": 28.29127023288229}, "sampler_perf": {"mean_env_wait_ms": 1568.7422574699535, "mean_processing_ms": 424.410110494769, "mean_inference_ms": 1.869180422676217}, "off_policy_estimator": {}, "info": {"num_steps_trained": 116066432, "num_steps_sampled": 1649000, "sample_time_ms": 2039861.81, "load_time_ms": 467.381, "grad_time_ms": 189199.319, "update_time_ms": 8.433, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4324.09033203125, "policy_loss": 0.0002998356649186462, "vf_loss": 4324.08984375, "vf_explained_var": 0.16980843245983124, "vf_preds": 109.96345520019531, "kl": 0.004206282552331686, "entropy": 0.41662418842315674, "entropy_coeff": 0.0, "advantages": -2.496957449693582e-06, "rewards": 1.8421027660369873}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1649000, "episodes_total": 1649, "training_iteration": 97, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_12-51-33", "timestamp": 1599223893, "time_this_iter_s": 2230.838953256607, "time_total_s": 214501.88622045517, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc44c158>", "on_episode_step": ".on_episode_step at 0x7f3afc44c1e0>", "on_episode_end": ".on_episode_end at 0x7f3afc44cb70>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc44c268>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 214501.88622045517, "timesteps_since_restore": 1649000, "iterations_since_restore": 97, "perf": {"cpu_util_percent": 87.98394711992445, "ram_util_percent": 30.540352533836955}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 141477.36294169014, "episode_reward_min": 139238.69197392688, "episode_reward_mean": 140376.28704627077, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 618.5481782232916}, "policy_reward_mean": {"av": 440.3547495020727}, "custom_metrics": {"avg_speed_mean": 4.9966965910205525, "avg_speed_min": 4.927213612249453, "avg_speed_max": 5.084463201775719, "avg_speed_avs_mean": 4.993999481239476, "avg_speed_avs_min": 4.922526641191299, "avg_speed_avs_max": 5.085005356390428, "avg_headway_avs_mean": 7.274906017954354, "avg_headway_avs_min": 6.804819740281156, "avg_headway_avs_max": 7.778997806489627, "std_headway_avs_mean": 6.584100669899346, "std_headway_avs_min": 6.083222494490728, "std_headway_avs_max": 7.396490272977906, "avg_accel_avs_mean": 0.4078869881852466, "avg_accel_avs_min": 0.39110864395883016, "avg_accel_avs_max": 0.4259836260480462, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 27.781191081210245, "avg_mpg_per_veh_min": 26.894016745009473, "avg_mpg_per_veh_max": 28.41661550534895, "num_cars_mean": 933.28168, "num_cars_min": 921.523, "num_cars_max": 942.886, "avg_mpg_mean": 27.657527908880606, "avg_mpg_min": 26.784684395208867, "avg_mpg_max": 28.29127023288229}, "sampler_perf": {"mean_env_wait_ms": 1569.347924057626, "mean_processing_ms": 424.17945578205274, "mean_inference_ms": 1.8694197862130522}, "off_policy_estimator": {}, "info": {"num_steps_trained": 117317504, "num_steps_sampled": 1666000, "sample_time_ms": 2040864.531, "load_time_ms": 467.624, "grad_time_ms": 188418.859, "update_time_ms": 8.174, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4330.458984375, "policy_loss": 0.00036147533683106303, "vf_loss": 4330.45849609375, "vf_explained_var": 0.1591954231262207, "vf_preds": 113.44786834716797, "kl": 0.003949995152652264, "entropy": 0.43893373012542725, "entropy_coeff": 0.0, "advantages": 1.202871590066934e-05, "rewards": 1.8375526666641235}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1666000, "episodes_total": 1666, "training_iteration": 98, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_13-28-50", "timestamp": 1599226130, "time_this_iter_s": 2236.880765914917, "time_total_s": 216738.7669863701, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc34e598>", "on_episode_step": ".on_episode_step at 0x7f3afc34e620>", "on_episode_end": ".on_episode_end at 0x7f3afc34e6a8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc34e730>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 216738.7669863701, "timesteps_since_restore": 1666000, "iterations_since_restore": 98, "perf": {"cpu_util_percent": 87.39155681104833, "ram_util_percent": 30.438794726930322}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 141477.36294169014, "episode_reward_min": 138905.5565942331, "episode_reward_mean": 140372.02899363497, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 618.5481782232916}, "policy_reward_mean": {"av": 440.34139216272973}, "custom_metrics": {"avg_speed_mean": 4.991421112178911, "avg_speed_min": 4.927213612249453, "avg_speed_max": 5.053608934283365, "avg_speed_avs_mean": 4.988690425207109, "avg_speed_avs_min": 4.922526641191299, "avg_speed_avs_max": 5.0531776979263645, "avg_headway_avs_mean": 7.254942814949221, "avg_headway_avs_min": 6.804819740281156, "avg_headway_avs_max": 7.778997806489627, "std_headway_avs_mean": 6.540462932575098, "std_headway_avs_min": 6.083222494490728, "std_headway_avs_max": 7.089965832520053, "avg_accel_avs_mean": 0.4077870063346588, "avg_accel_avs_min": 0.39110864395883016, "avg_accel_avs_max": 0.4258725176371822, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 27.799987078488456, "avg_mpg_per_veh_min": 27.01940103582292, "avg_mpg_per_veh_max": 28.41661550534895, "num_cars_mean": 933.8499300000002, "num_cars_min": 924.302, "num_cars_max": 942.886, "avg_mpg_mean": 27.67665290597076, "avg_mpg_min": 26.88310503379238, "avg_mpg_max": 28.29127023288229}, "sampler_perf": {"mean_env_wait_ms": 1569.8858883052133, "mean_processing_ms": 423.98849475986395, "mean_inference_ms": 1.8695730592362836}, "off_policy_estimator": {}, "info": {"num_steps_trained": 118562944, "num_steps_sampled": 1683000, "sample_time_ms": 2040633.818, "load_time_ms": 467.11, "grad_time_ms": 187359.043, "update_time_ms": 8.298, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4150.3798828125, "policy_loss": 0.0002567150804679841, "vf_loss": 4150.37939453125, "vf_explained_var": 0.2154775857925415, "vf_preds": 120.25850677490234, "kl": 0.004409408662468195, "entropy": 0.3839152157306671, "entropy_coeff": 0.0, "advantages": 7.578591066703666e-06, "rewards": 1.8445665836334229}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1683000, "episodes_total": 1683, "training_iteration": 99, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_14-05-56", "timestamp": 1599228356, "time_this_iter_s": 2226.162009716034, "time_total_s": 218964.92899608612, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5410d0>", "on_episode_step": ".on_episode_step at 0x7f3afc541048>", "on_episode_end": ".on_episode_end at 0x7f3afc5701e0>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc570158>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 218964.92899608612, "timesteps_since_restore": 1683000, "iterations_since_restore": 99, "perf": {"cpu_util_percent": 87.85490381583098, "ram_util_percent": 30.536013875748978}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 141477.36294169014, "episode_reward_min": 138905.5565942331, "episode_reward_mean": 140260.99326903553, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 618.5481782232916}, "policy_reward_mean": {"av": 440.0068804123208}, "custom_metrics": {"avg_speed_mean": 4.992077008425992, "avg_speed_min": 4.927213612249453, "avg_speed_max": 5.070428520032636, "avg_speed_avs_mean": 4.989131198068981, "avg_speed_avs_min": 4.922526641191299, "avg_speed_avs_max": 5.067285641287674, "avg_headway_avs_mean": 7.26169618607987, "avg_headway_avs_min": 6.692969570858454, "avg_headway_avs_max": 7.778997806489627, "std_headway_avs_mean": 6.5501312528418625, "std_headway_avs_min": 6.083222494490728, "std_headway_avs_max": 7.444544022352541, "avg_accel_avs_mean": 0.4087704670769446, "avg_accel_avs_min": 0.39110864395883016, "avg_accel_avs_max": 0.4258725176371822, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 27.737757840760484, "avg_mpg_per_veh_min": 26.885267947302868, "avg_mpg_per_veh_max": 28.41661550534895, "num_cars_mean": 933.70536, "num_cars_min": 921.863, "num_cars_max": 942.886, "avg_mpg_mean": 27.615967597116704, "avg_mpg_min": 26.747955368826325, "avg_mpg_max": 28.29127023288229}, "sampler_perf": {"mean_env_wait_ms": 1570.3777556325024, "mean_processing_ms": 423.81870011028894, "mean_inference_ms": 1.869658245675682}, "off_policy_estimator": {}, "info": {"num_steps_trained": 119808384, "num_steps_sampled": 1700000, "sample_time_ms": 2044504.419, "load_time_ms": 467.238, "grad_time_ms": 186703.896, "update_time_ms": 8.244, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4302.6728515625, "policy_loss": 0.00032139281393028796, "vf_loss": 4302.6728515625, "vf_explained_var": 0.23527230322360992, "vf_preds": 121.09049224853516, "kl": 0.004269599448889494, "entropy": 0.36702120304107666, "entropy_coeff": 0.0, "advantages": -2.4074188331724145e-06, "rewards": 1.8440051078796387}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1700000, "episodes_total": 1700, "training_iteration": 100, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_14-43-18", "timestamp": 1599230598, "time_this_iter_s": 2241.99351811409, "time_total_s": 221206.9225142002, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc570730>", "on_episode_step": ".on_episode_step at 0x7f3afc5706a8>", "on_episode_end": ".on_episode_end at 0x7f3afc627730>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc6279d8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 221206.9225142002, "timesteps_since_restore": 1700000, "iterations_since_restore": 100, "perf": {"cpu_util_percent": 88.83271759549153, "ram_util_percent": 30.590075140889166}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 141477.36294169014, "episode_reward_min": 138905.5565942331, "episode_reward_mean": 140178.55578228753, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 618.5481782232916}, "policy_reward_mean": {"av": 439.91387347336416}, "custom_metrics": {"avg_speed_mean": 4.989783521390414, "avg_speed_min": 4.927213612249453, "avg_speed_max": 5.070428520032636, "avg_speed_avs_mean": 4.986666721605695, "avg_speed_avs_min": 4.922526641191299, "avg_speed_avs_max": 5.067285641287674, "avg_headway_avs_mean": 7.246764992056151, "avg_headway_avs_min": 6.692969570858454, "avg_headway_avs_max": 7.778997806489627, "std_headway_avs_mean": 6.530081563517593, "std_headway_avs_min": 6.083222494490728, "std_headway_avs_max": 7.444544022352541, "avg_accel_avs_mean": 0.40961581206317904, "avg_accel_avs_min": 0.39110864395883016, "avg_accel_avs_max": 0.4258725176371822, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 27.693548866250136, "avg_mpg_per_veh_min": 26.885267947302868, "avg_mpg_per_veh_max": 28.41661550534895, "num_cars_mean": 933.78858, "num_cars_min": 921.863, "num_cars_max": 942.886, "avg_mpg_mean": 27.572604492542695, "avg_mpg_min": 26.747955368826325, "avg_mpg_max": 28.29127023288229}, "sampler_perf": {"mean_env_wait_ms": 1570.8070359183216, "mean_processing_ms": 423.6322714640286, "mean_inference_ms": 1.869686628910855}, "off_policy_estimator": {}, "info": {"num_steps_trained": 121056512, "num_steps_sampled": 1717000, "sample_time_ms": 2044498.558, "load_time_ms": 467.988, "grad_time_ms": 186014.908, "update_time_ms": 8.13, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4525.90673828125, "policy_loss": 0.0002644236374180764, "vf_loss": 4525.90625, "vf_explained_var": 0.20731744170188904, "vf_preds": 118.65103912353516, "kl": 0.0039434731006622314, "entropy": 0.37732887268066406, "entropy_coeff": 0.0, "advantages": 6.924695298948791e-06, "rewards": 1.8439363241195679}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1717000, "episodes_total": 1717, "training_iteration": 101, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_15-19-55", "timestamp": 1599232795, "time_this_iter_s": 2196.820908308029, "time_total_s": 223403.74342250824, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc3b09d8>", "on_episode_step": ".on_episode_step at 0x7f3afc3b0a60>", "on_episode_end": ".on_episode_end at 0x7f3afc3b0ae8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc3b0d08>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 223403.74342250824, "timesteps_since_restore": 1717000, "iterations_since_restore": 101, "perf": {"cpu_util_percent": 86.47092651757188, "ram_util_percent": 30.58006389776358}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 142000.41448172677, "episode_reward_min": 138905.5565942331, "episode_reward_mean": 140200.38470501485, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 618.5481782232916}, "policy_reward_mean": {"av": 439.8995472530353}, "custom_metrics": {"avg_speed_mean": 4.989840005076566, "avg_speed_min": 4.918927006697632, "avg_speed_max": 5.070428520032636, "avg_speed_avs_mean": 4.98697538565835, "avg_speed_avs_min": 4.918298862255751, "avg_speed_avs_max": 5.067285641287674, "avg_headway_avs_mean": 7.219774339020806, "avg_headway_avs_min": 6.629092348289082, "avg_headway_avs_max": 7.778997806489627, "std_headway_avs_mean": 6.503792275713687, "std_headway_avs_min": 5.826643565384186, "std_headway_avs_max": 7.444544022352541, "avg_accel_avs_mean": 0.4091188805161453, "avg_accel_avs_min": 0.39119872147099854, "avg_accel_avs_max": 0.4258725176371822, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 27.69533972299138, "avg_mpg_per_veh_min": 26.885267947302868, "avg_mpg_per_veh_max": 28.510343966076718, "num_cars_mean": 933.7859300000001, "num_cars_min": 921.863, "num_cars_max": 943.036, "avg_mpg_mean": 27.573874739868703, "avg_mpg_min": 26.747955368826325, "avg_mpg_max": 28.374612393486412}, "sampler_perf": {"mean_env_wait_ms": 1571.200957365619, "mean_processing_ms": 423.48035204816426, "mean_inference_ms": 1.8696940230262697}, "off_policy_estimator": {}, "info": {"num_steps_trained": 122307328, "num_steps_sampled": 1734000, "sample_time_ms": 2048303.165, "load_time_ms": 466.714, "grad_time_ms": 185322.74, "update_time_ms": 8.142, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4527.33740234375, "policy_loss": 0.00023132786736823618, "vf_loss": 4527.337890625, "vf_explained_var": 0.1923803836107254, "vf_preds": 118.20262145996094, "kl": 0.004187737591564655, "entropy": 0.3689548075199127, "entropy_coeff": 0.0, "advantages": -8.509975486958865e-06, "rewards": 1.8471627235412598}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1734000, "episodes_total": 1734, "training_iteration": 102, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_15-57-25", "timestamp": 1599235045, "time_this_iter_s": 2249.307678937912, "time_total_s": 225653.05110144615, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc672ae8>", "on_episode_step": ".on_episode_step at 0x7f3afc672048>", "on_episode_end": ".on_episode_end at 0x7f3afc672a60>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc6729d8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 225653.05110144615, "timesteps_since_restore": 1734000, "iterations_since_restore": 102, "perf": {"cpu_util_percent": 88.8861111111111, "ram_util_percent": 30.45474406991261}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 142000.41448172677, "episode_reward_min": 138905.5565942331, "episode_reward_mean": 140285.16057563783, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 618.5481782232916}, "policy_reward_mean": {"av": 439.95847887987776}, "custom_metrics": {"avg_speed_mean": 4.989974943717, "avg_speed_min": 4.918927006697632, "avg_speed_max": 5.070428520032636, "avg_speed_avs_mean": 4.987441272782655, "avg_speed_avs_min": 4.918298862255751, "avg_speed_avs_max": 5.067285641287674, "avg_headway_avs_mean": 7.179670967615591, "avg_headway_avs_min": 6.629092348289082, "avg_headway_avs_max": 7.674305079729006, "std_headway_avs_mean": 6.469721324991189, "std_headway_avs_min": 5.826643565384186, "std_headway_avs_max": 7.444544022352541, "avg_accel_avs_mean": 0.4070164387414416, "avg_accel_avs_min": 0.38286818581425097, "avg_accel_avs_max": 0.4258725176371822, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 27.735732271134157, "avg_mpg_per_veh_min": 26.885267947302868, "avg_mpg_per_veh_max": 28.637070156067907, "num_cars_mean": 933.8834000000002, "num_cars_min": 921.863, "num_cars_max": 944.202, "avg_mpg_mean": 27.61310180711177, "avg_mpg_min": 26.747955368826325, "avg_mpg_max": 28.52187186527628}, "sampler_perf": {"mean_env_wait_ms": 1571.5427008193062, "mean_processing_ms": 423.39664751710006, "mean_inference_ms": 1.869633964988177}, "off_policy_estimator": {}, "info": {"num_steps_trained": 123559552, "num_steps_sampled": 1751000, "sample_time_ms": 2049565.974, "load_time_ms": 468.668, "grad_time_ms": 184506.977, "update_time_ms": 7.626, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4566.6259765625, "policy_loss": 0.0001315160043304786, "vf_loss": 4566.6259765625, "vf_explained_var": 0.17039239406585693, "vf_preds": 120.17926788330078, "kl": 0.004563924390822649, "entropy": 0.3582702577114105, "entropy_coeff": 0.0, "advantages": -2.31604531109042e-06, "rewards": 1.8471876382827759}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1751000, "episodes_total": 1751, "training_iteration": 103, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_16-34-50", "timestamp": 1599237290, "time_this_iter_s": 2245.1040029525757, "time_total_s": 227898.15510439873, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc627488>", "on_episode_step": ".on_episode_step at 0x7f3afc627f28>", "on_episode_end": ".on_episode_end at 0x7f3afc627d90>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc6277b8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 227898.15510439873, "timesteps_since_restore": 1751000, "iterations_since_restore": 103, "perf": {"cpu_util_percent": 88.49224515322076, "ram_util_percent": 30.56160100062539}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 142000.41448172677, "episode_reward_min": 138907.9390058129, "episode_reward_mean": 140443.9224735347, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 615.5092925151653}, "policy_reward_mean": {"av": 440.318292179379}, "custom_metrics": {"avg_speed_mean": 4.992671380687325, "avg_speed_min": 4.918927006697632, "avg_speed_max": 5.070428520032636, "avg_speed_avs_mean": 4.9904015529822825, "avg_speed_avs_min": 4.918298862255751, "avg_speed_avs_max": 5.067285641287674, "avg_headway_avs_mean": 7.177989312682389, "avg_headway_avs_min": 6.629092348289082, "avg_headway_avs_max": 7.810768828595763, "std_headway_avs_mean": 6.479021643360913, "std_headway_avs_min": 5.826643565384186, "std_headway_avs_max": 7.444544022352541, "avg_accel_avs_mean": 0.4025589557839181, "avg_accel_avs_min": 0.3797636296031312, "avg_accel_avs_max": 0.4258725176371822, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 27.83586312411429, "avg_mpg_per_veh_min": 26.885267947302868, "avg_mpg_per_veh_max": 28.813446481715836, "num_cars_mean": 933.84206, "num_cars_min": 921.863, "num_cars_max": 944.202, "avg_mpg_mean": 27.712942264341013, "avg_mpg_min": 26.747955368826325, "avg_mpg_max": 28.68973206283003}, "sampler_perf": {"mean_env_wait_ms": 1571.799842731111, "mean_processing_ms": 423.3954244439512, "mean_inference_ms": 1.8694832108941097}, "off_policy_estimator": {}, "info": {"num_steps_trained": 124809984, "num_steps_sampled": 1768000, "sample_time_ms": 2047697.149, "load_time_ms": 466.524, "grad_time_ms": 184281.739, "update_time_ms": 7.733, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4605.4365234375, "policy_loss": 0.00020815333118662238, "vf_loss": 4605.4365234375, "vf_explained_var": 0.17670945823192596, "vf_preds": 120.32369232177734, "kl": 0.004484731238335371, "entropy": 0.32911553978919983, "entropy_coeff": 0.0, "advantages": 7.233449650811963e-06, "rewards": 1.8503926992416382}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1768000, "episodes_total": 1768, "training_iteration": 104, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_17-11-55", "timestamp": 1599239515, "time_this_iter_s": 2225.036206007004, "time_total_s": 230123.19131040573, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc570a60>", "on_episode_step": ".on_episode_step at 0x7f3afc570ae8>", "on_episode_end": ".on_episode_end at 0x7f3afc570378>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc570488>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 230123.19131040573, "timesteps_since_restore": 1768000, "iterations_since_restore": 104, "perf": {"cpu_util_percent": 87.89438308614704, "ram_util_percent": 30.53480593247081}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 142000.41448172677, "episode_reward_min": 139125.65533292704, "episode_reward_mean": 140576.46337264767, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 615.5092925151653}, "policy_reward_mean": {"av": 440.63712933782926}, "custom_metrics": {"avg_speed_mean": 4.990540488553579, "avg_speed_min": 4.918927006697632, "avg_speed_max": 5.070428520032636, "avg_speed_avs_mean": 4.988422315969142, "avg_speed_avs_min": 4.918298862255751, "avg_speed_avs_max": 5.067285641287674, "avg_headway_avs_mean": 7.181619565926886, "avg_headway_avs_min": 6.629092348289082, "avg_headway_avs_max": 7.810768828595763, "std_headway_avs_mean": 6.457886863251009, "std_headway_avs_min": 5.826643565384186, "std_headway_avs_max": 7.444544022352541, "avg_accel_avs_mean": 0.3982782020322865, "avg_accel_avs_min": 0.3797636296031312, "avg_accel_avs_max": 0.4215128350265479, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 27.928142089328702, "avg_mpg_per_veh_min": 26.885267947302868, "avg_mpg_per_veh_max": 28.813446481715836, "num_cars_mean": 934.31445, "num_cars_min": 921.863, "num_cars_max": 944.202, "avg_mpg_mean": 27.80450153756285, "avg_mpg_min": 26.747955368826325, "avg_mpg_max": 28.68973206283003}, "sampler_perf": {"mean_env_wait_ms": 1571.982060649406, "mean_processing_ms": 423.471810427939, "mean_inference_ms": 1.8692462168512831}, "off_policy_estimator": {}, "info": {"num_steps_trained": 126057984, "num_steps_sampled": 1785000, "sample_time_ms": 2047708.674, "load_time_ms": 468.312, "grad_time_ms": 184199.514, "update_time_ms": 7.686, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4520.046875, "policy_loss": 0.00018736501806415617, "vf_loss": 4520.04638671875, "vf_explained_var": 0.22427359223365784, "vf_preds": 121.08563995361328, "kl": 0.004475337918847799, "entropy": 0.29325705766677856, "entropy_coeff": 0.0, "advantages": -4.081829047208885e-06, "rewards": 1.8505818843841553}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1785000, "episodes_total": 1785, "training_iteration": 105, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_17-49-08", "timestamp": 1599241748, "time_this_iter_s": 2233.0595021247864, "time_total_s": 232356.25081253052, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5417b8>", "on_episode_step": ".on_episode_step at 0x7f3afc541d08>", "on_episode_end": ".on_episode_end at 0x7f3afc541e18>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc672510>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 232356.25081253052, "timesteps_since_restore": 1785000, "iterations_since_restore": 105, "perf": {"cpu_util_percent": 87.94185534591195, "ram_util_percent": 30.583113207547168}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 142000.41448172677, "episode_reward_min": 138037.10858899416, "episode_reward_mean": 140506.5782640736, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 615.5092925151653}, "policy_reward_mean": {"av": 440.40426988488474}, "custom_metrics": {"avg_speed_mean": 4.995490768573945, "avg_speed_min": 4.918927006697632, "avg_speed_max": 5.065241980879431, "avg_speed_avs_mean": 4.993840965327067, "avg_speed_avs_min": 4.918298862255751, "avg_speed_avs_max": 5.066472565878311, "avg_headway_avs_mean": 7.201351529303542, "avg_headway_avs_min": 6.629092348289082, "avg_headway_avs_max": 7.810768828595763, "std_headway_avs_mean": 6.524915658478303, "std_headway_avs_min": 5.826643565384186, "std_headway_avs_max": 7.3620725757133805, "avg_accel_avs_mean": 0.39612039925424747, "avg_accel_avs_min": 0.3797636296031312, "avg_accel_avs_max": 0.4201979519448755, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 27.889109902053093, "avg_mpg_per_veh_min": 26.52474831665135, "avg_mpg_per_veh_max": 28.813446481715836, "num_cars_mean": 933.44944, "num_cars_min": 921.443, "num_cars_max": 944.202, "avg_mpg_mean": 27.764248993089705, "avg_mpg_min": 26.407141375690095, "avg_mpg_max": 28.68973206283003}, "sampler_perf": {"mean_env_wait_ms": 1572.0318485685816, "mean_processing_ms": 423.61404569519493, "mean_inference_ms": 1.868881343980851}, "off_policy_estimator": {}, "info": {"num_steps_trained": 127297920, "num_steps_sampled": 1802000, "sample_time_ms": 2043018.107, "load_time_ms": 470.619, "grad_time_ms": 183874.939, "update_time_ms": 7.552, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4283.4248046875, "policy_loss": 0.00026035436894744635, "vf_loss": 4283.4248046875, "vf_explained_var": 0.2880277633666992, "vf_preds": 125.43622589111328, "kl": 0.004848403390496969, "entropy": 0.22047372162342072, "entropy_coeff": 0.0, "advantages": 1.0658740393409971e-05, "rewards": 1.8487673997879028}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1802000, "episodes_total": 1802, "training_iteration": 106, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_18-25-41", "timestamp": 1599243941, "time_this_iter_s": 2192.519749879837, "time_total_s": 234548.77056241035, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc3b02f0>", "on_episode_step": ".on_episode_step at 0x7f3afc3b07b8>", "on_episode_end": ".on_episode_end at 0x7f3afc3b0840>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc3b0268>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 234548.77056241035, "timesteps_since_restore": 1802000, "iterations_since_restore": 106, "perf": {"cpu_util_percent": 87.16019852705732, "ram_util_percent": 30.538840858149218}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 142000.41448172677, "episode_reward_min": 138037.10858899416, "episode_reward_mean": 140412.4438910404, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 615.5092925151653}, "policy_reward_mean": {"av": 440.12301003366576}, "custom_metrics": {"avg_speed_mean": 5.002877992357848, "avg_speed_min": 4.918927006697632, "avg_speed_max": 5.068565079370647, "avg_speed_avs_mean": 5.001648712065179, "avg_speed_avs_min": 4.918298862255751, "avg_speed_avs_max": 5.067051865936899, "avg_headway_avs_mean": 7.203336169698446, "avg_headway_avs_min": 6.629092348289082, "avg_headway_avs_max": 7.810768828595763, "std_headway_avs_mean": 6.610640346003204, "std_headway_avs_min": 5.826643565384186, "std_headway_avs_max": 7.499529644756755, "avg_accel_avs_mean": 0.39414772953482, "avg_accel_avs_min": 0.3797636296031312, "avg_accel_avs_max": 0.412025943520917, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 27.839169699946392, "avg_mpg_per_veh_min": 26.52474831665135, "avg_mpg_per_veh_max": 28.813446481715836, "num_cars_mean": 932.15255, "num_cars_min": 920.313, "num_cars_max": 944.202, "avg_mpg_mean": 27.71304495503765, "avg_mpg_min": 26.407141375690095, "avg_mpg_max": 28.68973206283003}, "sampler_perf": {"mean_env_wait_ms": 1572.0640397997895, "mean_processing_ms": 423.8351160836152, "mean_inference_ms": 1.8685094695237574}, "off_policy_estimator": {}, "info": {"num_steps_trained": 128537856, "num_steps_sampled": 1819000, "sample_time_ms": 2044174.336, "load_time_ms": 470.407, "grad_time_ms": 183554.902, "update_time_ms": 7.657, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4257.67041015625, "policy_loss": 0.000335603574058041, "vf_loss": 4257.669921875, "vf_explained_var": 0.3098964989185333, "vf_preds": 125.43807220458984, "kl": 0.004599960055202246, "entropy": 0.19994479417800903, "entropy_coeff": 0.0, "advantages": 3.750810037672636e-06, "rewards": 1.8502860069274902}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1819000, "episodes_total": 1819, "training_iteration": 107, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_19-03-00", "timestamp": 1599246180, "time_this_iter_s": 2239.170897245407, "time_total_s": 236787.94145965576, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc627d90>", "on_episode_step": ".on_episode_step at 0x7f3afc627488>", "on_episode_end": ".on_episode_end at 0x7f3afc627ae8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc627840>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 236787.94145965576, "timesteps_since_restore": 1819000, "iterations_since_restore": 107, "perf": {"cpu_util_percent": 89.43877742946707, "ram_util_percent": 30.4566144200627}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 141658.88882589358, "episode_reward_min": 138037.10858899416, "episode_reward_mean": 140260.66639377148, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 615.5092925151653}, "policy_reward_mean": {"av": 439.6886093848636}, "custom_metrics": {"avg_speed_mean": 5.01132399175925, "avg_speed_min": 4.935007008793304, "avg_speed_max": 5.072027327524076, "avg_speed_avs_mean": 5.010044008402121, "avg_speed_avs_min": 4.9342374498976875, "avg_speed_avs_max": 5.07098231974899, "avg_headway_avs_mean": 7.206315154055319, "avg_headway_avs_min": 6.833032249975673, "avg_headway_avs_max": 7.810768828595763, "std_headway_avs_mean": 6.6945566756264965, "std_headway_avs_min": 5.943878240740036, "std_headway_avs_max": 7.499529644756755, "avg_accel_avs_mean": 0.3950967272522574, "avg_accel_avs_min": 0.3797636296031312, "avg_accel_avs_max": 0.41269126485495927, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 27.728768648125666, "avg_mpg_per_veh_min": 26.52474831665135, "avg_mpg_per_veh_max": 28.813446481715836, "num_cars_mean": 930.5864600000001, "num_cars_min": 919.904, "num_cars_max": 944.202, "avg_mpg_mean": 27.601242691955072, "avg_mpg_min": 26.407141375690095, "avg_mpg_max": 28.68973206283003}, "sampler_perf": {"mean_env_wait_ms": 1571.9706389883513, "mean_processing_ms": 424.08506967259774, "mean_inference_ms": 1.868009674808282}, "off_policy_estimator": {}, "info": {"num_steps_trained": 129779840, "num_steps_sampled": 1836000, "sample_time_ms": 2040566.494, "load_time_ms": 468.037, "grad_time_ms": 183118.706, "update_time_ms": 7.752, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4347.35546875, "policy_loss": 0.00016146432608366013, "vf_loss": 4347.35546875, "vf_explained_var": 0.3036971688270569, "vf_preds": 123.6913070678711, "kl": 0.004966232925653458, "entropy": 0.24409136176109314, "entropy_coeff": 0.0, "advantages": 6.957857863199024e-07, "rewards": 1.8478223085403442}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1836000, "episodes_total": 1836, "training_iteration": 108, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_19-39-36", "timestamp": 1599248376, "time_this_iter_s": 2196.4051265716553, "time_total_s": 238984.34658622742, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc75f6a8>", "on_episode_step": ".on_episode_step at 0x7f3afc75f8c8>", "on_episode_end": ".on_episode_end at 0x7f3afc75f0d0>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc75fd90>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 238984.34658622742, "timesteps_since_restore": 1836000, "iterations_since_restore": 108, "perf": {"cpu_util_percent": 86.51684782608694, "ram_util_percent": 30.43634910485933}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 141658.88882589358, "episode_reward_min": 137650.6677277789, "episode_reward_mean": 139845.20425714145, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 615.5092925151653}, "policy_reward_mean": {"av": 438.57869992203916}, "custom_metrics": {"avg_speed_mean": 5.02758897870774, "avg_speed_min": 4.943816548206043, "avg_speed_max": 5.101793572970876, "avg_speed_avs_mean": 5.026931276044379, "avg_speed_avs_min": 4.941556119764344, "avg_speed_avs_max": 5.10522736564282, "avg_headway_avs_mean": 7.197373137552494, "avg_headway_avs_min": 6.833032249975673, "avg_headway_avs_max": 7.810768828595763, "std_headway_avs_mean": 6.85349567784145, "std_headway_avs_min": 6.10793841648733, "std_headway_avs_max": 7.6947333296187095, "avg_accel_avs_mean": 0.39970047899671185, "avg_accel_avs_min": 0.3797636296031312, "avg_accel_avs_max": 0.43136114646917156, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 27.409095852092687, "avg_mpg_per_veh_min": 25.86452704837956, "avg_mpg_per_veh_max": 28.813446481715836, "num_cars_mean": 927.8055899999999, "num_cars_min": 913.738, "num_cars_max": 940.675, "avg_mpg_mean": 27.280540704751417, "avg_mpg_min": 25.737598860025447, "avg_mpg_max": 28.68973206283003}, "sampler_perf": {"mean_env_wait_ms": 1571.8131703144682, "mean_processing_ms": 424.3346521133206, "mean_inference_ms": 1.8674788396261248}, "off_policy_estimator": {}, "info": {"num_steps_trained": 131012608, "num_steps_sampled": 1853000, "sample_time_ms": 2039011.111, "load_time_ms": 468.666, "grad_time_ms": 182973.854, "update_time_ms": 7.732, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4186.1123046875, "policy_loss": 0.0003769852628465742, "vf_loss": 4186.11181640625, "vf_explained_var": 0.37386849522590637, "vf_preds": 128.9665985107422, "kl": 0.0045633139088749886, "entropy": 0.17132988572120667, "entropy_coeff": 0.0, "advantages": -8.936921744862047e-07, "rewards": 1.8431756496429443}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1853000, "episodes_total": 1853, "training_iteration": 109, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_20-16-26", "timestamp": 1599250586, "time_this_iter_s": 2209.154307603836, "time_total_s": 241193.50089383125, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc570bf8>", "on_episode_step": ".on_episode_step at 0x7f3afc570598>", "on_episode_end": ".on_episode_end at 0x7f3afc570840>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc570488>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 241193.50089383125, "timesteps_since_restore": 1853000, "iterations_since_restore": 109, "perf": {"cpu_util_percent": 88.03666984429616, "ram_util_percent": 30.486177311725452}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 141369.0440737569, "episode_reward_min": 136311.61367287577, "episode_reward_mean": 139297.40663032696, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 613.6525811253921}, "policy_reward_mean": {"av": 436.97034516069675}, "custom_metrics": {"avg_speed_mean": 5.045938317608576, "avg_speed_min": 4.952381221017605, "avg_speed_max": 5.1248923800992765, "avg_speed_avs_mean": 5.045734917529508, "avg_speed_avs_min": 4.953071369265161, "avg_speed_avs_max": 5.127117211562893, "avg_headway_avs_mean": 7.187506858366809, "avg_headway_avs_min": 6.833032249975673, "avg_headway_avs_max": 7.504279975901079, "std_headway_avs_mean": 7.039512249083059, "std_headway_avs_min": 6.10793841648733, "std_headway_avs_max": 7.981449419290877, "avg_accel_avs_mean": 0.40616895853390383, "avg_accel_avs_min": 0.3849788893677551, "avg_accel_avs_max": 0.43136114646917156, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 27.0049503294279, "avg_mpg_per_veh_min": 25.66072112739476, "avg_mpg_per_veh_max": 28.191190323157397, "num_cars_mean": 924.50207, "num_cars_min": 909.939, "num_cars_max": 938.682, "avg_mpg_mean": 26.875954116410266, "avg_mpg_min": 25.527661040141666, "avg_mpg_max": 28.063369483467202}, "sampler_perf": {"mean_env_wait_ms": 1571.659257701299, "mean_processing_ms": 424.5796744368092, "mean_inference_ms": 1.8669988069597676}, "off_policy_estimator": {}, "info": {"num_steps_trained": 132241024, "num_steps_sampled": 1870000, "sample_time_ms": 2038073.312, "load_time_ms": 467.081, "grad_time_ms": 182552.645, "update_time_ms": 7.951, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4221.44091796875, "policy_loss": 0.0002987154002767056, "vf_loss": 4221.43994140625, "vf_explained_var": 0.41543546319007874, "vf_preds": 131.1848907470703, "kl": 0.005392810329794884, "entropy": 0.12361767888069153, "entropy_coeff": 0.0, "advantages": 1.580091407049622e-06, "rewards": 1.8411120176315308}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1870000, "episodes_total": 1870, "training_iteration": 110, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_20-53-34", "timestamp": 1599252814, "time_this_iter_s": 2228.364312171936, "time_total_s": 243421.8652060032, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc627f28>", "on_episode_step": ".on_episode_step at 0x7f3afc627d08>", "on_episode_end": ".on_episode_end at 0x7f3afc627a60>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc6272f0>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 243421.8652060032, "timesteps_since_restore": 1870000, "iterations_since_restore": 110, "perf": {"cpu_util_percent": 89.63396345305607, "ram_util_percent": 30.622621298046628}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 141369.0440737569, "episode_reward_min": 136311.61367287577, "episode_reward_mean": 139025.7963274092, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 613.6525811253921}, "policy_reward_mean": {"av": 436.3099307287507}, "custom_metrics": {"avg_speed_mean": 5.0584806611307815, "avg_speed_min": 4.995922904122117, "avg_speed_max": 5.1248923800992765, "avg_speed_avs_mean": 5.058506340193727, "avg_speed_avs_min": 4.996717504598288, "avg_speed_avs_max": 5.127117211562893, "avg_headway_avs_mean": 7.210580078715067, "avg_headway_avs_min": 6.873500363111615, "avg_headway_avs_max": 7.65429984045902, "std_headway_avs_mean": 7.171515364460752, "std_headway_avs_min": 6.4661917325199125, "std_headway_avs_max": 7.981449419290877, "avg_accel_avs_mean": 0.4087287198296968, "avg_accel_avs_min": 0.3883656226454196, "avg_accel_avs_max": 0.43136114646917156, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 26.808716576504036, "avg_mpg_per_veh_min": 25.66072112739476, "avg_mpg_per_veh_max": 27.968474549620034, "num_cars_mean": 922.45228, "num_cars_min": 909.939, "num_cars_max": 931.686, "avg_mpg_mean": 26.678937012686923, "avg_mpg_min": 25.527661040141666, "avg_mpg_max": 27.844318056192307}, "sampler_perf": {"mean_env_wait_ms": 1571.4640775560372, "mean_processing_ms": 424.8160279520344, "mean_inference_ms": 1.8664957992319173}, "off_policy_estimator": {}, "info": {"num_steps_trained": 133472128, "num_steps_sampled": 1887000, "sample_time_ms": 2038192.037, "load_time_ms": 470.988, "grad_time_ms": 182108.57, "update_time_ms": 7.949, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4346.32470703125, "policy_loss": 0.00021586356160696596, "vf_loss": 4346.32421875, "vf_explained_var": 0.41474881768226624, "vf_preds": 128.38775634765625, "kl": 0.0052767242304980755, "entropy": 0.11220980435609818, "entropy_coeff": 0.0, "advantages": 7.892747817095369e-06, "rewards": 1.851744294166565}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1887000, "episodes_total": 1887, "training_iteration": 111, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_21-30-08", "timestamp": 1599255008, "time_this_iter_s": 2193.5973329544067, "time_total_s": 245615.4625389576, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3b003b3d90>", "on_episode_step": ".on_episode_step at 0x7f3b003b3f28>", "on_episode_end": ".on_episode_end at 0x7f3b003b3a60>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3b003b3730>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 245615.4625389576, "timesteps_since_restore": 1887000, "iterations_since_restore": 111, "perf": {"cpu_util_percent": 87.60816, "ram_util_percent": 30.635392000000003}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 140877.41197684614, "episode_reward_min": 136311.61367287577, "episode_reward_mean": 138906.87025960852, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 613.6525811253921}, "policy_reward_mean": {"av": 435.82727867598044}, "custom_metrics": {"avg_speed_mean": 5.063790937236692, "avg_speed_min": 4.995922904122117, "avg_speed_max": 5.1248923800992765, "avg_speed_avs_mean": 5.063971624626361, "avg_speed_avs_min": 4.996717504598288, "avg_speed_avs_max": 5.127117211562893, "avg_headway_avs_mean": 7.21212440428423, "avg_headway_avs_min": 6.873500363111615, "avg_headway_avs_max": 7.65429984045902, "std_headway_avs_mean": 7.232580528371949, "std_headway_avs_min": 6.4661917325199125, "std_headway_avs_max": 7.981449419290877, "avg_accel_avs_mean": 0.4100355990679164, "avg_accel_avs_min": 0.3916712414349053, "avg_accel_avs_max": 0.43136114646917156, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 26.708034090561394, "avg_mpg_per_veh_min": 25.66072112739476, "avg_mpg_per_veh_max": 27.74557217864217, "num_cars_mean": 921.53886, "num_cars_min": 909.939, "num_cars_max": 931.686, "avg_mpg_mean": 26.575954445439205, "avg_mpg_min": 25.527661040141666, "avg_mpg_max": 27.607145316318203}, "sampler_perf": {"mean_env_wait_ms": 1571.3096914329135, "mean_processing_ms": 425.04877793304934, "mean_inference_ms": 1.8660517963425767}, "off_policy_estimator": {}, "info": {"num_steps_trained": 134704896, "num_steps_sampled": 1904000, "sample_time_ms": 2035680.567, "load_time_ms": 469.773, "grad_time_ms": 182018.329, "update_time_ms": 8.184, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4427.4033203125, "policy_loss": 0.00027709282585419714, "vf_loss": 4427.4033203125, "vf_explained_var": 0.4138520658016205, "vf_preds": 129.5063018798828, "kl": 0.005250751506537199, "entropy": 0.1186210885643959, "entropy_coeff": 0.0, "advantages": 2.876871258195024e-05, "rewards": 1.8511368036270142}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1904000, "episodes_total": 1904, "training_iteration": 112, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_22-07-11", "timestamp": 1599257231, "time_this_iter_s": 2223.25634765625, "time_total_s": 247838.71888661385, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc3b0158>", "on_episode_step": ".on_episode_step at 0x7f3afc3b0378>", "on_episode_end": ".on_episode_end at 0x7f3afc3b0400>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc3b0620>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 247838.71888661385, "timesteps_since_restore": 1904000, "iterations_since_restore": 112, "perf": {"cpu_util_percent": 89.02757422615286, "ram_util_percent": 30.601200252684773}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 140877.41197684614, "episode_reward_min": 136311.61367287577, "episode_reward_mean": 138665.35717575162, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 613.2274000305787}, "policy_reward_mean": {"av": 435.0831702041091}, "custom_metrics": {"avg_speed_mean": 5.071164984996423, "avg_speed_min": 5.010010388656115, "avg_speed_max": 5.1248923800992765, "avg_speed_avs_mean": 5.071456285717983, "avg_speed_avs_min": 5.005787750161727, "avg_speed_avs_max": 5.127117211562893, "avg_headway_avs_mean": 7.2278657601743745, "avg_headway_avs_min": 6.873500363111615, "avg_headway_avs_max": 7.65429984045902, "std_headway_avs_mean": 7.317578196313157, "std_headway_avs_min": 6.4661917325199125, "std_headway_avs_max": 7.981449419290877, "avg_accel_avs_mean": 0.4131026088121477, "avg_accel_avs_min": 0.39502881324360084, "avg_accel_avs_max": 0.43136114646917156, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 26.52590839786751, "avg_mpg_per_veh_min": 25.66072112739476, "avg_mpg_per_veh_max": 27.669192140240263, "num_cars_mean": 920.2958799999999, "num_cars_min": 909.939, "num_cars_max": 930.343, "avg_mpg_mean": 26.392121460186463, "avg_mpg_min": 25.527661040141666, "avg_mpg_max": 27.527362842071867}, "sampler_perf": {"mean_env_wait_ms": 1571.1117625798463, "mean_processing_ms": 425.2756465792255, "mean_inference_ms": 1.8655922474175533}, "off_policy_estimator": {}, "info": {"num_steps_trained": 135935232, "num_steps_sampled": 1921000, "sample_time_ms": 2031630.213, "load_time_ms": 464.074, "grad_time_ms": 181878.75, "update_time_ms": 8.326, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4478.73583984375, "policy_loss": 0.000360961101250723, "vf_loss": 4478.7353515625, "vf_explained_var": 0.43310797214508057, "vf_preds": 129.25294494628906, "kl": 0.005284077022224665, "entropy": 0.0986362174153328, "entropy_coeff": 0.0, "advantages": 5.1160313887521625e-06, "rewards": 1.846045970916748}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1921000, "episodes_total": 1921, "training_iteration": 113, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_22-43-54", "timestamp": 1599259434, "time_this_iter_s": 2203.1316163539886, "time_total_s": 250041.85050296783, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5708c8>", "on_episode_step": ".on_episode_step at 0x7f3afc570400>", "on_episode_end": ".on_episode_end at 0x7f3afc5702f0>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc570620>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 250041.85050296783, "timesteps_since_restore": 1921000, "iterations_since_restore": 113, "perf": {"cpu_util_percent": 88.0671551449506, "ram_util_percent": 30.530264415418923}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 140100.3302383502, "episode_reward_min": 136311.61367287577, "episode_reward_mean": 138597.9047968368, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 613.2274000305787}, "policy_reward_mean": {"av": 435.03532689926493}, "custom_metrics": {"avg_speed_mean": 5.077279393902711, "avg_speed_min": 5.0325231640551875, "avg_speed_max": 5.1248923800992765, "avg_speed_avs_mean": 5.077860743256917, "avg_speed_avs_min": 5.030999874661482, "avg_speed_avs_max": 5.127117211562893, "avg_headway_avs_mean": 7.251418445748759, "avg_headway_avs_min": 6.883197371120659, "avg_headway_avs_max": 7.65429984045902, "std_headway_avs_mean": 7.393201363739644, "std_headway_avs_min": 6.745251041501303, "std_headway_avs_max": 7.981449419290877, "avg_accel_avs_mean": 0.41292406322787806, "avg_accel_avs_min": 0.39502881324360084, "avg_accel_avs_max": 0.43136114646917156, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 26.479635915849148, "avg_mpg_per_veh_min": 25.66072112739476, "avg_mpg_per_veh_max": 27.47168237328718, "num_cars_mean": 919.2871900000001, "num_cars_min": 909.939, "num_cars_max": 927.749, "avg_mpg_mean": 26.344068517096275, "avg_mpg_min": 25.527661040141666, "avg_mpg_max": 27.315856433753325}, "sampler_perf": {"mean_env_wait_ms": 1570.9455865621217, "mean_processing_ms": 425.49872889150635, "mean_inference_ms": 1.8651949743855893}, "off_policy_estimator": {}, "info": {"num_steps_trained": 137168128, "num_steps_sampled": 1938000, "sample_time_ms": 2030148.068, "load_time_ms": 459.243, "grad_time_ms": 181989.649, "update_time_ms": 8.409, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4506.609375, "policy_loss": 0.00041173826321028173, "vf_loss": 4506.6083984375, "vf_explained_var": 0.42334890365600586, "vf_preds": 123.6967544555664, "kl": 0.005167506635189056, "entropy": 0.09355927258729935, "entropy_coeff": 0.0, "advantages": -7.522301530116238e-06, "rewards": 1.8537094593048096}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1938000, "episodes_total": 1938, "training_iteration": 114, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_23-20-46", "timestamp": 1599261646, "time_this_iter_s": 2211.2735822200775, "time_total_s": 252253.1240851879, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc6c0048>", "on_episode_step": ".on_episode_step at 0x7f3afc6c0d08>", "on_episode_end": ".on_episode_end at 0x7f3afc6c0d90>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc6c0e18>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 252253.1240851879, "timesteps_since_restore": 1938000, "iterations_since_restore": 114, "perf": {"cpu_util_percent": 88.19180692283264, "ram_util_percent": 30.501651317878697}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 140545.58515632586, "episode_reward_min": 136311.61367287577, "episode_reward_mean": 138839.3620492305, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 613.2274000305787}, "policy_reward_mean": {"av": 435.5744691740567}, "custom_metrics": {"avg_speed_mean": 5.076104948677863, "avg_speed_min": 5.0325231640551875, "avg_speed_max": 5.1248923800992765, "avg_speed_avs_mean": 5.076460416135951, "avg_speed_avs_min": 5.030569913556072, "avg_speed_avs_max": 5.127117211562893, "avg_headway_avs_mean": 7.254503683125723, "avg_headway_avs_min": 6.883197371120659, "avg_headway_avs_max": 7.65429984045902, "std_headway_avs_mean": 7.365075446395526, "std_headway_avs_min": 6.745251041501303, "std_headway_avs_max": 7.981449419290877, "avg_accel_avs_mean": 0.4093651251409389, "avg_accel_avs_min": 0.3903711977937156, "avg_accel_avs_max": 0.4313101550317376, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 26.645571418650785, "avg_mpg_per_veh_min": 25.66072112739476, "avg_mpg_per_veh_max": 27.856715525738768, "num_cars_mean": 919.5781199999999, "num_cars_min": 909.939, "num_cars_max": 927.656, "avg_mpg_mean": 26.510134794611876, "avg_mpg_min": 25.527661040141666, "avg_mpg_max": 27.74210065751999}, "sampler_perf": {"mean_env_wait_ms": 1570.7624506691145, "mean_processing_ms": 425.71915838844893, "mean_inference_ms": 1.8647877941913533}, "off_policy_estimator": {}, "info": {"num_steps_trained": 138404480, "num_steps_sampled": 1955000, "sample_time_ms": 2025777.752, "load_time_ms": 456.849, "grad_time_ms": 182079.386, "update_time_ms": 8.413, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4404.52001953125, "policy_loss": 0.00042861574911512434, "vf_loss": 4404.51953125, "vf_explained_var": 0.4150068163871765, "vf_preds": 117.72010803222656, "kl": 0.0050167664885520935, "entropy": 0.11403777450323105, "entropy_coeff": 0.0, "advantages": -2.089383769998676e-06, "rewards": 1.856935739517212}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1955000, "episodes_total": 1955, "training_iteration": 115, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-04_23-57-16", "timestamp": 1599263836, "time_this_iter_s": 2190.3448989391327, "time_total_s": 254443.46898412704, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5e99d8>", "on_episode_step": ".on_episode_step at 0x7f3afc5e9a60>", "on_episode_end": ".on_episode_end at 0x7f3afc5e9ae8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5e98c8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 254443.46898412704, "timesteps_since_restore": 1955000, "iterations_since_restore": 115, "perf": {"cpu_util_percent": 87.2060576923077, "ram_util_percent": 30.52955128205128}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 140545.58515632586, "episode_reward_min": 137756.04577612408, "episode_reward_mean": 139088.07756641365, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 613.2274000305787}, "policy_reward_mean": {"av": 436.368443139906}, "custom_metrics": {"avg_speed_mean": 5.074328498542393, "avg_speed_min": 5.0325231640551875, "avg_speed_max": 5.118521591378607, "avg_speed_avs_mean": 5.074904666806676, "avg_speed_avs_min": 5.030569913556072, "avg_speed_avs_max": 5.119972629424038, "avg_headway_avs_mean": 7.259378522304572, "avg_headway_avs_min": 6.883197371120659, "avg_headway_avs_max": 7.65429984045902, "std_headway_avs_mean": 7.355796458007717, "std_headway_avs_min": 6.745251041501303, "std_headway_avs_max": 7.840812994368038, "avg_accel_avs_mean": 0.40566660996192505, "avg_accel_avs_min": 0.3903711977937156, "avg_accel_avs_max": 0.42455317801397935, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 26.797541112438747, "avg_mpg_per_veh_min": 25.983379079064413, "avg_mpg_per_veh_max": 27.856715525738768, "num_cars_mean": 920.1101999999998, "num_cars_min": 913.243, "num_cars_max": 927.656, "avg_mpg_mean": 26.659615857558514, "avg_mpg_min": 25.824696731043716, "avg_mpg_max": 27.74210065751999}, "sampler_perf": {"mean_env_wait_ms": 1570.5538102588246, "mean_processing_ms": 425.9338371200489, "mean_inference_ms": 1.864342276421612}, "off_policy_estimator": {}, "info": {"num_steps_trained": 139636352, "num_steps_sampled": 1972000, "sample_time_ms": 2026608.207, "load_time_ms": 459.069, "grad_time_ms": 182203.634, "update_time_ms": 8.381, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4150.57470703125, "policy_loss": 0.0003771920164581388, "vf_loss": 4150.57421875, "vf_explained_var": 0.4508152902126312, "vf_preds": 121.09510040283203, "kl": 0.005173543002456427, "entropy": 0.05405121669173241, "entropy_coeff": 0.0, "advantages": 3.68559113894662e-07, "rewards": 1.8567519187927246}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1972000, "episodes_total": 1972, "training_iteration": 116, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_00-33-58", "timestamp": 1599266038, "time_this_iter_s": 2202.053571462631, "time_total_s": 256645.52255558968, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc6c0d08>", "on_episode_step": ".on_episode_step at 0x7f3afc3b0400>", "on_episode_end": ".on_episode_end at 0x7f3afc3b0048>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc3b07b8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 256645.52255558968, "timesteps_since_restore": 1972000, "iterations_since_restore": 116, "perf": {"cpu_util_percent": 88.2780044628626, "ram_util_percent": 30.582562958240356}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 140545.58515632586, "episode_reward_min": 137756.04577612408, "episode_reward_mean": 139206.40216418094, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 616.8624798194559}, "policy_reward_mean": {"av": 436.71226679690346}, "custom_metrics": {"avg_speed_mean": 5.070599435251374, "avg_speed_min": 5.027827567132863, "avg_speed_max": 5.118521591378607, "avg_speed_avs_mean": 5.071083100846343, "avg_speed_avs_min": 5.027943754567057, "avg_speed_avs_max": 5.119972629424038, "avg_headway_avs_mean": 7.240993537433619, "avg_headway_avs_min": 6.883197371120659, "avg_headway_avs_max": 7.630438161085773, "std_headway_avs_mean": 7.324820691442742, "std_headway_avs_min": 6.745251041501303, "std_headway_avs_max": 7.840812994368038, "avg_accel_avs_mean": 0.4035324264985758, "avg_accel_avs_min": 0.38981780476622996, "avg_accel_avs_max": 0.42455317801397935, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 26.879078052556228, "avg_mpg_per_veh_min": 25.983379079064413, "avg_mpg_per_veh_max": 27.856715525738768, "num_cars_mean": 920.6195000000001, "num_cars_min": 913.243, "num_cars_max": 929.569, "avg_mpg_mean": 26.742229474361302, "avg_mpg_min": 25.824696731043716, "avg_mpg_max": 27.74210065751999}, "sampler_perf": {"mean_env_wait_ms": 1570.3630284474677, "mean_processing_ms": 426.14755705336853, "mean_inference_ms": 1.8639103893154276}, "off_policy_estimator": {}, "info": {"num_steps_trained": 140871680, "num_steps_sampled": 1989000, "sample_time_ms": 2022745.185, "load_time_ms": 461.859, "grad_time_ms": 182420.661, "update_time_ms": 8.362, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4261.43505859375, "policy_loss": 0.0004557197680696845, "vf_loss": 4261.4345703125, "vf_explained_var": 0.4390527904033661, "vf_preds": 116.50018310546875, "kl": 0.005429170560091734, "entropy": 0.05375451222062111, "entropy_coeff": 0.0, "advantages": -8.898066880647093e-06, "rewards": 1.854670524597168}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 1989000, "episodes_total": 1989, "training_iteration": 117, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_01-10-41", "timestamp": 1599268241, "time_this_iter_s": 2202.762418270111, "time_total_s": 258848.2849738598, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc75fe18>", "on_episode_step": ".on_episode_step at 0x7f3afc627158>", "on_episode_end": ".on_episode_end at 0x7f3afc627ae8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc6276a8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 258848.2849738598, "timesteps_since_restore": 1989000, "iterations_since_restore": 117, "perf": {"cpu_util_percent": 88.01724577621931, "ram_util_percent": 30.6003506534906}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 142161.5520116894, "episode_reward_min": 137756.04577612408, "episode_reward_mean": 139625.2165150619, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 616.8624798194559}, "policy_reward_mean": {"av": 437.8613162163257}, "custom_metrics": {"avg_speed_mean": 5.0639970574214495, "avg_speed_min": 5.004998924928098, "avg_speed_max": 5.118521591378607, "avg_speed_avs_mean": 5.064386393621302, "avg_speed_avs_min": 5.009711926266939, "avg_speed_avs_max": 5.119972629424038, "avg_headway_avs_mean": 7.221359364446268, "avg_headway_avs_min": 6.883197371120659, "avg_headway_avs_max": 7.630438161085773, "std_headway_avs_mean": 7.2290879688824985, "std_headway_avs_min": 6.486536473161313, "std_headway_avs_max": 7.840812994368038, "avg_accel_avs_mean": 0.39741865032345947, "avg_accel_avs_min": 0.36899957040977577, "avg_accel_avs_max": 0.42279029934679263, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 27.16909824226928, "avg_mpg_per_veh_min": 25.983379079064413, "avg_mpg_per_veh_max": 28.94723139078796, "num_cars_mean": 922.27352, "num_cars_min": 913.243, "num_cars_max": 935.657, "avg_mpg_mean": 27.030947812225914, "avg_mpg_min": 25.824696731043716, "avg_mpg_max": 28.760114439007875}, "sampler_perf": {"mean_env_wait_ms": 1570.113049654899, "mean_processing_ms": 426.3538453839815, "mean_inference_ms": 1.8633968454564072}, "off_policy_estimator": {}, "info": {"num_steps_trained": 142115968, "num_steps_sampled": 2006000, "sample_time_ms": 2020816.953, "load_time_ms": 461.98, "grad_time_ms": 182696.333, "update_time_ms": 8.251, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4333.369140625, "policy_loss": 0.0003931251703761518, "vf_loss": 4333.36865234375, "vf_explained_var": 0.3698006868362427, "vf_preds": 105.27438354492188, "kl": 0.0056638699024915695, "entropy": 0.1230122521519661, "entropy_coeff": 0.0, "advantages": -8.210069609049242e-06, "rewards": 1.8661770820617676}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2006000, "episodes_total": 2006, "training_iteration": 118, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_01-47-01", "timestamp": 1599270421, "time_this_iter_s": 2179.887074947357, "time_total_s": 261028.17204880714, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5e9a60>", "on_episode_step": ".on_episode_step at 0x7f3afc5e9ae8>", "on_episode_end": ".on_episode_end at 0x7f3afc5e98c8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5e9400>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 261028.17204880714, "timesteps_since_restore": 2006000, "iterations_since_restore": 118, "perf": {"cpu_util_percent": 85.76064412238325, "ram_util_percent": 30.64148148148148}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 142161.5520116894, "episode_reward_min": 138301.9745995814, "episode_reward_mean": 139982.83837114964, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 616.8624798194559}, "policy_reward_mean": {"av": 438.900226911487}, "custom_metrics": {"avg_speed_mean": 5.059666611702564, "avg_speed_min": 5.004998924928098, "avg_speed_max": 5.118521591378607, "avg_speed_avs_mean": 5.060176653389966, "avg_speed_avs_min": 5.009711926266939, "avg_speed_avs_max": 5.119972629424038, "avg_headway_avs_mean": 7.201865909216947, "avg_headway_avs_min": 6.828136332997035, "avg_headway_avs_max": 7.630438161085773, "std_headway_avs_mean": 7.173734853662541, "std_headway_avs_min": 6.486536473161313, "std_headway_avs_max": 7.840812994368038, "avg_accel_avs_mean": 0.39168602227566374, "avg_accel_avs_min": 0.36899957040977577, "avg_accel_avs_max": 0.410428287942433, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 27.415110287202964, "avg_mpg_per_veh_min": 26.40022020732895, "avg_mpg_per_veh_max": 28.94723139078796, "num_cars_mean": 923.3227400000001, "num_cars_min": 914.707, "num_cars_max": 935.657, "avg_mpg_mean": 27.27579328265371, "avg_mpg_min": 26.265895350477503, "avg_mpg_max": 28.760114439007875}, "sampler_perf": {"mean_env_wait_ms": 1569.9055566308994, "mean_processing_ms": 426.55823851231713, "mean_inference_ms": 1.8629101669958752}, "off_policy_estimator": {}, "info": {"num_steps_trained": 143355264, "num_steps_sampled": 2023000, "sample_time_ms": 2023091.708, "load_time_ms": 460.943, "grad_time_ms": 183043.856, "update_time_ms": 8.04, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3940.60595703125, "policy_loss": 0.0004944282700307667, "vf_loss": 3940.605712890625, "vf_explained_var": 0.42117348313331604, "vf_preds": 105.51472473144531, "kl": 0.005066731944680214, "entropy": 0.0588616319000721, "entropy_coeff": 0.0, "advantages": -2.2260467176238308e-06, "rewards": 1.8627139329910278}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2023000, "episodes_total": 2023, "training_iteration": 119, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_02-24-16", "timestamp": 1599272656, "time_this_iter_s": 2235.40992808342, "time_total_s": 263263.58197689056, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc3b0048>", "on_episode_step": ".on_episode_step at 0x7f3afc3b08c8>", "on_episode_end": ".on_episode_end at 0x7f3afc3b06a8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc3b0158>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 263263.58197689056, "timesteps_since_restore": 2023000, "iterations_since_restore": 119, "perf": {"cpu_util_percent": 89.0650863422292, "ram_util_percent": 30.556797488226064}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 142376.5095789266, "episode_reward_min": 138731.49513314286, "episode_reward_mean": 140370.94898727248, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 616.8624798194559}, "policy_reward_mean": {"av": 439.8412890495473}, "custom_metrics": {"avg_speed_mean": 5.054353664795295, "avg_speed_min": 5.004998924928098, "avg_speed_max": 5.118521591378607, "avg_speed_avs_mean": 5.0552207247167305, "avg_speed_avs_min": 5.009711926266939, "avg_speed_avs_max": 5.119972629424038, "avg_headway_avs_mean": 7.175674386188446, "avg_headway_avs_min": 6.828136332997035, "avg_headway_avs_max": 7.630438161085773, "std_headway_avs_mean": 7.095724437318974, "std_headway_avs_min": 6.486536473161313, "std_headway_avs_max": 7.840812994368038, "avg_accel_avs_mean": 0.38464251473187977, "avg_accel_avs_min": 0.356798105691317, "avg_accel_avs_max": 0.4090573571885544, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 27.671811175665322, "avg_mpg_per_veh_min": 26.62614825553697, "avg_mpg_per_veh_max": 28.94723139078796, "num_cars_mean": 924.8301700000001, "num_cars_min": 915.545, "num_cars_max": 935.657, "avg_mpg_mean": 27.532275048801807, "avg_mpg_min": 26.495487816398253, "avg_mpg_max": 28.764491367043036}, "sampler_perf": {"mean_env_wait_ms": 1569.7204899398248, "mean_processing_ms": 426.7597051268877, "mean_inference_ms": 1.8624472601784023}, "off_policy_estimator": {}, "info": {"num_steps_trained": 144598784, "num_steps_sampled": 2040000, "sample_time_ms": 2021527.759, "load_time_ms": 465.562, "grad_time_ms": 183406.702, "update_time_ms": 7.784, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4060.363525390625, "policy_loss": 0.0004751723899971694, "vf_loss": 4060.363037109375, "vf_explained_var": 0.3867197632789612, "vf_preds": 101.84815979003906, "kl": 0.0051162466406822205, "entropy": 0.055135712027549744, "entropy_coeff": 0.0, "advantages": -3.87933914680616e-06, "rewards": 1.868462324142456}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2040000, "episodes_total": 2040, "training_iteration": 120, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_03-01-13", "timestamp": 1599274873, "time_this_iter_s": 2216.4248611927032, "time_total_s": 265480.00683808327, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5701e0>", "on_episode_step": ".on_episode_step at 0x7f3afc570598>", "on_episode_end": ".on_episode_end at 0x7f3afc570488>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5709d8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 265480.00683808327, "timesteps_since_restore": 2040000, "iterations_since_restore": 120, "perf": {"cpu_util_percent": 88.1690528983212, "ram_util_percent": 30.466265441875198}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 142941.72216353528, "episode_reward_min": 138787.37558580938, "episode_reward_mean": 140823.0866309825, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 617.9493190919786}, "policy_reward_mean": {"av": 441.0231017850443}, "custom_metrics": {"avg_speed_mean": 5.046316807987499, "avg_speed_min": 5.0020946215835265, "avg_speed_max": 5.099850532459488, "avg_speed_avs_mean": 5.0472681088546425, "avg_speed_avs_min": 4.999663328200473, "avg_speed_avs_max": 5.103246036322535, "avg_headway_avs_mean": 7.148822438664094, "avg_headway_avs_min": 6.828136332997035, "avg_headway_avs_max": 7.526921637473286, "std_headway_avs_mean": 6.998527374908664, "std_headway_avs_min": 6.306063511853804, "std_headway_avs_max": 7.840812994368038, "avg_accel_avs_mean": 0.3769100902309878, "avg_accel_avs_min": 0.34696056460950503, "avg_accel_avs_max": 0.404544378416664, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 27.998237149354367, "avg_mpg_per_veh_min": 26.677587839395883, "avg_mpg_per_veh_max": 29.484756324197228, "num_cars_mean": 926.94196, "num_cars_min": 915.858, "num_cars_max": 939.593, "avg_mpg_mean": 27.85373697392231, "avg_mpg_min": 26.5551505432119, "avg_mpg_max": 29.32281463286158}, "sampler_perf": {"mean_env_wait_ms": 1569.5986216432407, "mean_processing_ms": 426.95645279436167, "mean_inference_ms": 1.8620402035188928}, "off_policy_estimator": {}, "info": {"num_steps_trained": 145846912, "num_steps_sampled": 2057000, "sample_time_ms": 2025929.147, "load_time_ms": 461.383, "grad_time_ms": 183842.291, "update_time_ms": 8.091, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4041.2978515625, "policy_loss": 0.00044387945672497153, "vf_loss": 4041.297607421875, "vf_explained_var": 0.3440449833869934, "vf_preds": 99.01972961425781, "kl": 0.004521502181887627, "entropy": 0.10372715443372726, "entropy_coeff": 0.0, "advantages": 1.2615600098797586e-05, "rewards": 1.8734420537948608}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2057000, "episodes_total": 2057, "training_iteration": 121, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_03-38-35", "timestamp": 1599277115, "time_this_iter_s": 2241.95582818985, "time_total_s": 267721.9626662731, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc6c07b8>", "on_episode_step": ".on_episode_step at 0x7f3afc6c0378>", "on_episode_end": ".on_episode_end at 0x7f3afc6c0d90>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc6c0e18>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 267721.9626662731, "timesteps_since_restore": 2057000, "iterations_since_restore": 121, "perf": {"cpu_util_percent": 88.40497964296898, "ram_util_percent": 30.446789852803008}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 142941.72216353528, "episode_reward_min": 138787.37558580938, "episode_reward_mean": 141343.0354307762, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 618.6205374286969}, "policy_reward_mean": {"av": 442.2774749069912}, "custom_metrics": {"avg_speed_mean": 5.038886422176731, "avg_speed_min": 4.985390929219103, "avg_speed_max": 5.099603103150815, "avg_speed_avs_mean": 5.039928142176973, "avg_speed_avs_min": 4.986551062499956, "avg_speed_avs_max": 5.0984806620518075, "avg_headway_avs_mean": 7.122817249103876, "avg_headway_avs_min": 6.708958577717569, "avg_headway_avs_max": 7.526921637473286, "std_headway_avs_mean": 6.875348776012118, "std_headway_avs_min": 6.15634774971275, "std_headway_avs_max": 7.5703873246856315, "avg_accel_avs_mean": 0.368777163241841, "avg_accel_avs_min": 0.3440772329407073, "avg_accel_avs_max": 0.398828756525934, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 28.362759349903264, "avg_mpg_per_veh_min": 26.677587839395883, "avg_mpg_per_veh_max": 29.484756324197228, "num_cars_mean": 929.0711900000001, "num_cars_min": 915.858, "num_cars_max": 939.713, "avg_mpg_mean": 28.215176028088425, "avg_mpg_min": 26.5551505432119, "avg_mpg_max": 29.32281463286158}, "sampler_perf": {"mean_env_wait_ms": 1569.5004514374536, "mean_processing_ms": 427.15119816408645, "mean_inference_ms": 1.861661676718177}, "off_policy_estimator": {}, "info": {"num_steps_trained": 147093632, "num_steps_sampled": 2074000, "sample_time_ms": 2026077.331, "load_time_ms": 465.65, "grad_time_ms": 184100.404, "update_time_ms": 8.044, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3769.259033203125, "policy_loss": 0.0002473845088388771, "vf_loss": 3769.2587890625, "vf_explained_var": 0.3506608307361603, "vf_preds": 102.30316925048828, "kl": 0.005365284159779549, "entropy": 0.0885637104511261, "entropy_coeff": 0.0, "advantages": -1.1787521998485317e-06, "rewards": 1.8759294748306274}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2074000, "episodes_total": 2074, "training_iteration": 122, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_04-15-42", "timestamp": 1599279342, "time_this_iter_s": 2227.358634710312, "time_total_s": 269949.3213009834, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc3b02f0>", "on_episode_step": ".on_episode_step at 0x7f3afc3b0400>", "on_episode_end": ".on_episode_end at 0x7f3afc3b08c8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc3b0048>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 269949.3213009834, "timesteps_since_restore": 2074000, "iterations_since_restore": 122, "perf": {"cpu_util_percent": 87.7716356760164, "ram_util_percent": 30.47251812165143}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 143912.44138391272, "episode_reward_min": 139735.71845037478, "episode_reward_mean": 141950.2149049808, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 618.6205374286969}, "policy_reward_mean": {"av": 443.80245397836745}, "custom_metrics": {"avg_speed_mean": 5.032058878285255, "avg_speed_min": 4.985390929219103, "avg_speed_max": 5.099603103150815, "avg_speed_avs_mean": 5.033235231807163, "avg_speed_avs_min": 4.986551062499956, "avg_speed_avs_max": 5.0984806620518075, "avg_headway_avs_mean": 7.101827311604511, "avg_headway_avs_min": 6.708958577717569, "avg_headway_avs_max": 7.481496557459765, "std_headway_avs_mean": 6.75194724346556, "std_headway_avs_min": 6.105336587370791, "std_headway_avs_max": 7.33600891621458, "avg_accel_avs_mean": 0.35903310520370385, "avg_accel_avs_min": 0.32511231511805233, "avg_accel_avs_max": 0.3871362553807454, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 28.788196827208036, "avg_mpg_per_veh_min": 27.233113914365372, "avg_mpg_per_veh_max": 30.170343480014598, "num_cars_mean": 931.4379100000001, "num_cars_min": 920.427, "num_cars_max": 941.556, "avg_mpg_mean": 28.636472578900733, "avg_mpg_min": 27.07892837012186, "avg_mpg_max": 30.023686402327346}, "sampler_perf": {"mean_env_wait_ms": 1569.4458122381027, "mean_processing_ms": 427.34316516678797, "mean_inference_ms": 1.861319075409138}, "off_policy_estimator": {}, "info": {"num_steps_trained": 148344192, "num_steps_sampled": 2091000, "sample_time_ms": 2029725.395, "load_time_ms": 469.852, "grad_time_ms": 184531.265, "update_time_ms": 8.092, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3875.78955078125, "policy_loss": 0.0004116164636798203, "vf_loss": 3875.788818359375, "vf_explained_var": 0.3052324652671814, "vf_preds": 103.72445678710938, "kl": 0.004649339243769646, "entropy": 0.07908633351325989, "entropy_coeff": 0.0, "advantages": -2.6819466256711166e-06, "rewards": 1.8818631172180176}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2091000, "episodes_total": 2091, "training_iteration": 123, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_04-53-06", "timestamp": 1599281586, "time_this_iter_s": 2244.014065027237, "time_total_s": 272193.33536601067, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc75f9d8>", "on_episode_step": ".on_episode_step at 0x7f3afc6c0400>", "on_episode_end": ".on_episode_end at 0x7f3afc6c0510>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc570a60>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 272193.33536601067, "timesteps_since_restore": 2091000, "iterations_since_restore": 123, "perf": {"cpu_util_percent": 87.79518147684607, "ram_util_percent": 30.548372966207758}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 143912.44138391272, "episode_reward_min": 139735.71845037478, "episode_reward_mean": 142089.40619595814, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 618.6205374286969}, "policy_reward_mean": {"av": 444.2237422496035}, "custom_metrics": {"avg_speed_mean": 5.03276250149976, "avg_speed_min": 4.985390929219103, "avg_speed_max": 5.099603103150815, "avg_speed_avs_mean": 5.0339724949453695, "avg_speed_avs_min": 4.986551062499956, "avg_speed_avs_max": 5.0984806620518075, "avg_headway_avs_mean": 7.115615400458365, "avg_headway_avs_min": 6.708958577717569, "avg_headway_avs_max": 7.511341872174157, "std_headway_avs_mean": 6.755181658401735, "std_headway_avs_min": 6.105336587370791, "std_headway_avs_max": 7.33600891621458, "avg_accel_avs_mean": 0.35498462734653863, "avg_accel_avs_min": 0.32511231511805233, "avg_accel_avs_max": 0.3871362553807454, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 28.89885147627152, "avg_mpg_per_veh_min": 27.233113914365372, "avg_mpg_per_veh_max": 30.170343480014598, "num_cars_mean": 931.6379000000001, "num_cars_min": 920.427, "num_cars_max": 941.556, "avg_mpg_mean": 28.746572027619457, "avg_mpg_min": 27.07892837012186, "avg_mpg_max": 30.023686402327346}, "sampler_perf": {"mean_env_wait_ms": 1569.462411641633, "mean_processing_ms": 427.532725452077, "mean_inference_ms": 1.8610557022822223}, "off_policy_estimator": {}, "info": {"num_steps_trained": 149588224, "num_steps_sampled": 2108000, "sample_time_ms": 2031843.289, "load_time_ms": 470.64, "grad_time_ms": 184785.603, "update_time_ms": 7.914, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3749.99072265625, "policy_loss": 0.00023327850794885308, "vf_loss": 3749.990234375, "vf_explained_var": 0.3604142963886261, "vf_preds": 111.1106185913086, "kl": 0.0052795796655118465, "entropy": 0.0180975291877985, "entropy_coeff": 0.0, "advantages": 4.337051905167755e-06, "rewards": 1.8764886856079102}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2108000, "episodes_total": 2108, "training_iteration": 124, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_05-30-21", "timestamp": 1599283821, "time_this_iter_s": 2235.0115344524384, "time_total_s": 274428.3469004631, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5e9e18>", "on_episode_step": ".on_episode_step at 0x7f3afc5e9950>", "on_episode_end": ".on_episode_end at 0x7f3afc5e90d0>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5e9268>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 274428.3469004631, "timesteps_since_restore": 2108000, "iterations_since_restore": 124, "perf": {"cpu_util_percent": 88.78256361922715, "ram_util_percent": 30.60942507068803}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 143912.44138391272, "episode_reward_min": 140521.142254705, "episode_reward_mean": 142402.47056322356, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 618.6205374286969}, "policy_reward_mean": {"av": 445.0772638325474}, "custom_metrics": {"avg_speed_mean": 5.027013568026195, "avg_speed_min": 4.985390929219103, "avg_speed_max": 5.082034385642894, "avg_speed_avs_mean": 5.028251427896094, "avg_speed_avs_min": 4.986551062499956, "avg_speed_avs_max": 5.082352983643937, "avg_headway_avs_mean": 7.109553430523406, "avg_headway_avs_min": 6.708958577717569, "avg_headway_avs_max": 7.511341872174157, "std_headway_avs_mean": 6.68042973777828, "std_headway_avs_min": 6.105336587370791, "std_headway_avs_max": 7.313520155324042, "avg_accel_avs_mean": 0.3490066812121638, "avg_accel_avs_min": 0.32511231511805233, "avg_accel_avs_max": 0.36930849082645195, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 29.136817320894114, "avg_mpg_per_veh_min": 27.851493015106694, "avg_mpg_per_veh_max": 30.170343480014598, "num_cars_mean": 933.1187899999999, "num_cars_min": 922.358, "num_cars_max": 941.556, "avg_mpg_mean": 28.9810996127102, "avg_mpg_min": 27.726986935807545, "avg_mpg_max": 30.023686402327346}, "sampler_perf": {"mean_env_wait_ms": 1569.4845948792122, "mean_processing_ms": 427.71898675848223, "mean_inference_ms": 1.8607834564629606}, "off_policy_estimator": {}, "info": {"num_steps_trained": 150836736, "num_steps_sampled": 2125000, "sample_time_ms": 2037160.291, "load_time_ms": 477.945, "grad_time_ms": 185061.635, "update_time_ms": 7.963, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3929.425048828125, "policy_loss": 0.00038059038342908025, "vf_loss": 3929.424560546875, "vf_explained_var": 0.3264133036136627, "vf_preds": 109.96061706542969, "kl": 0.005208237562328577, "entropy": 0.039958517998456955, "entropy_coeff": 0.0, "advantages": -5.248031811788678e-06, "rewards": 1.8759945631027222}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2125000, "episodes_total": 2125, "training_iteration": 125, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_06-07-48", "timestamp": 1599286068, "time_this_iter_s": 2246.2361555099487, "time_total_s": 276674.58305597305, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc6c0048>", "on_episode_step": ".on_episode_step at 0x7f3afc6c0268>", "on_episode_end": ".on_episode_end at 0x7f3afc6c07b8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc6c0d08>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 276674.58305597305, "timesteps_since_restore": 2125000, "iterations_since_restore": 125, "perf": {"cpu_util_percent": 88.31609375, "ram_util_percent": 30.66221875}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144015.92021613938, "episode_reward_min": 141387.56036371744, "episode_reward_mean": 142701.3954607302, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 618.6205374286969}, "policy_reward_mean": {"av": 445.80254751868216}, "custom_metrics": {"avg_speed_mean": 5.021460395640782, "avg_speed_min": 4.973467590879922, "avg_speed_max": 5.082034385642894, "avg_speed_avs_mean": 5.0225797948769815, "avg_speed_avs_min": 4.973081053262882, "avg_speed_avs_max": 5.082352983643937, "avg_headway_avs_mean": 7.078916932978533, "avg_headway_avs_min": 6.559694280327167, "avg_headway_avs_max": 7.511341872174157, "std_headway_avs_mean": 6.585512386924513, "std_headway_avs_min": 5.765621376636129, "std_headway_avs_max": 7.313520155324042, "avg_accel_avs_mean": 0.3439506517725615, "avg_accel_avs_min": 0.32340959715932593, "avg_accel_avs_max": 0.36196571519776816, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 29.367641370015317, "avg_mpg_per_veh_min": 28.42972533122021, "avg_mpg_per_veh_max": 30.339968866156013, "num_cars_mean": 934.6733000000002, "num_cars_min": 922.358, "num_cars_max": 946.002, "avg_mpg_mean": 29.210122454723, "avg_mpg_min": 28.265785742610294, "avg_mpg_max": 30.18602389187769}, "sampler_perf": {"mean_env_wait_ms": 1569.5198742043026, "mean_processing_ms": 427.90117194909266, "mean_inference_ms": 1.8605047105958048}, "off_policy_estimator": {}, "info": {"num_steps_trained": 152090624, "num_steps_sampled": 2142000, "sample_time_ms": 2040491.239, "load_time_ms": 474.591, "grad_time_ms": 185469.864, "update_time_ms": 7.898, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4010.429443359375, "policy_loss": 0.00030873483046889305, "vf_loss": 4010.429443359375, "vf_explained_var": 0.2857009172439575, "vf_preds": 107.23698425292969, "kl": 0.004806274548172951, "entropy": 0.062061794102191925, "entropy_coeff": 0.0, "advantages": 1.8551629182184115e-05, "rewards": 1.8768970966339111}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2142000, "episodes_total": 2142, "training_iteration": 126, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_06-45-07", "timestamp": 1599288307, "time_this_iter_s": 2239.425773859024, "time_total_s": 278914.0088298321, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc672f28>", "on_episode_step": ".on_episode_step at 0x7f3afc6722f0>", "on_episode_end": ".on_episode_end at 0x7f3afc672048>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc672ae8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 278914.0088298321, "timesteps_since_restore": 2142000, "iterations_since_restore": 126, "perf": {"cpu_util_percent": 87.8247962382445, "ram_util_percent": 30.429686520376176}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144015.92021613938, "episode_reward_min": 141387.56036371744, "episode_reward_mean": 142753.86767336653, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 619.7305787470108}, "policy_reward_mean": {"av": 445.82719448271854}, "custom_metrics": {"avg_speed_mean": 5.019926583637146, "avg_speed_min": 4.973467590879922, "avg_speed_max": 5.082034385642894, "avg_speed_avs_mean": 5.021060517100891, "avg_speed_avs_min": 4.973076030684588, "avg_speed_avs_max": 5.082352983643937, "avg_headway_avs_mean": 7.059616153958604, "avg_headway_avs_min": 6.5413449946133895, "avg_headway_avs_max": 7.511341872174157, "std_headway_avs_mean": 6.560389288034728, "std_headway_avs_min": 5.765621376636129, "std_headway_avs_max": 7.25143153317172, "avg_accel_avs_mean": 0.3423151362720123, "avg_accel_avs_min": 0.32340959715932593, "avg_accel_avs_max": 0.35822878499858424, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 29.420069891094272, "avg_mpg_per_veh_min": 28.42972533122021, "avg_mpg_per_veh_max": 30.339968866156013, "num_cars_mean": 934.96782, "num_cars_min": 922.358, "num_cars_max": 946.002, "avg_mpg_mean": 29.262050536096893, "avg_mpg_min": 28.265785742610294, "avg_mpg_max": 30.18602389187769}, "sampler_perf": {"mean_env_wait_ms": 1569.5804203456314, "mean_processing_ms": 428.081612995129, "mean_inference_ms": 1.8602625694288522}, "off_policy_estimator": {}, "info": {"num_steps_trained": 153340928, "num_steps_sampled": 2159000, "sample_time_ms": 2046236.762, "load_time_ms": 475.88, "grad_time_ms": 185648.266, "update_time_ms": 8.249, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3997.072021484375, "policy_loss": 0.0003220992803107947, "vf_loss": 3997.07177734375, "vf_explained_var": 0.3093112111091614, "vf_preds": 111.77188110351562, "kl": 0.005278559401631355, "entropy": 0.05578816309571266, "entropy_coeff": 0.0, "advantages": -7.890451342973392e-06, "rewards": 1.8745179176330566}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2159000, "episodes_total": 2159, "training_iteration": 127, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_07-22-49", "timestamp": 1599290569, "time_this_iter_s": 2262.0697481632233, "time_total_s": 281176.0785779953, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc3b0950>", "on_episode_step": ".on_episode_step at 0x7f3afc3b06a8>", "on_episode_end": ".on_episode_end at 0x7f3afc3b08c8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc3b0488>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 281176.0785779953, "timesteps_since_restore": 2159000, "iterations_since_restore": 127, "perf": {"cpu_util_percent": 89.01598386095593, "ram_util_percent": 30.419584109248913}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144015.92021613938, "episode_reward_min": 140416.06022217526, "episode_reward_mean": 142541.73804782095, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 619.7305787470108}, "policy_reward_mean": {"av": 445.16470345977785}, "custom_metrics": {"avg_speed_mean": 5.022692885656561, "avg_speed_min": 4.973467590879922, "avg_speed_max": 5.082034385642894, "avg_speed_avs_mean": 5.02371274057466, "avg_speed_avs_min": 4.973076030684588, "avg_speed_avs_max": 5.082352983643937, "avg_headway_avs_mean": 7.054982477535674, "avg_headway_avs_min": 6.5413449946133895, "avg_headway_avs_max": 7.511341872174157, "std_headway_avs_mean": 6.616530648807022, "std_headway_avs_min": 5.765621376636129, "std_headway_avs_max": 7.35516444389472, "avg_accel_avs_mean": 0.34455532591455373, "avg_accel_avs_min": 0.32340959715932593, "avg_accel_avs_max": 0.3683431608061374, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 29.289900956041212, "avg_mpg_per_veh_min": 27.915550211094285, "avg_mpg_per_veh_max": 30.339968866156013, "num_cars_mean": 933.8753899999999, "num_cars_min": 922.358, "num_cars_max": 946.002, "avg_mpg_mean": 29.13181468327628, "avg_mpg_min": 27.77290129152728, "avg_mpg_max": 30.18602389187769}, "sampler_perf": {"mean_env_wait_ms": 1569.620334679788, "mean_processing_ms": 428.25853129597004, "mean_inference_ms": 1.8600024209308625}, "off_policy_estimator": {}, "info": {"num_steps_trained": 154583680, "num_steps_sampled": 2176000, "sample_time_ms": 2049562.23, "load_time_ms": 476.688, "grad_time_ms": 185646.182, "update_time_ms": 8.01, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 3907.53076171875, "policy_loss": 0.00034362092264927924, "vf_loss": 3907.53076171875, "vf_explained_var": 0.3752487897872925, "vf_preds": 118.09273529052734, "kl": 0.005326789803802967, "entropy": -0.004359293729066849, "entropy_coeff": 0.0, "advantages": 5.160311047802679e-06, "rewards": 1.8665999174118042}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2176000, "episodes_total": 2176, "training_iteration": 128, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_07-59-43", "timestamp": 1599292783, "time_this_iter_s": 2213.131589412689, "time_total_s": 283389.210167408, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc3b0048>", "on_episode_step": ".on_episode_step at 0x7f3afc3b0378>", "on_episode_end": ".on_episode_end at 0x7f3afc3b0620>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc3b0268>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 283389.210167408, "timesteps_since_restore": 2176000, "iterations_since_restore": 128, "perf": {"cpu_util_percent": 87.43540609137057, "ram_util_percent": 30.44739847715736}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144015.92021613938, "episode_reward_min": 140416.06022217526, "episode_reward_mean": 142412.87489912537, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 619.7305787470108}, "policy_reward_mean": {"av": 444.8734065323169}, "custom_metrics": {"avg_speed_mean": 5.022111515835166, "avg_speed_min": 4.973467590879922, "avg_speed_max": 5.082034385642894, "avg_speed_avs_mean": 5.023154263117239, "avg_speed_avs_min": 4.973076030684588, "avg_speed_avs_max": 5.082352983643937, "avg_headway_avs_mean": 7.027725654537061, "avg_headway_avs_min": 6.5413449946133895, "avg_headway_avs_max": 7.511341872174157, "std_headway_avs_mean": 6.634823168935776, "std_headway_avs_min": 5.765621376636129, "std_headway_avs_max": 7.35516444389472, "avg_accel_avs_mean": 0.34664110182739755, "avg_accel_avs_min": 0.32340959715932593, "avg_accel_avs_max": 0.3683431608061374, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 29.211152104803247, "avg_mpg_per_veh_min": 27.915550211094285, "avg_mpg_per_veh_max": 30.339968866156013, "num_cars_mean": 933.584, "num_cars_min": 922.358, "num_cars_max": 946.002, "avg_mpg_mean": 29.053350109827587, "avg_mpg_min": 27.77290129152728, "avg_mpg_max": 30.18602389187769}, "sampler_perf": {"mean_env_wait_ms": 1569.6714609084424, "mean_processing_ms": 428.43344611406764, "mean_inference_ms": 1.8597495875244636}, "off_policy_estimator": {}, "info": {"num_steps_trained": 155834112, "num_steps_sampled": 2193000, "sample_time_ms": 2049980.685, "load_time_ms": 477.921, "grad_time_ms": 185740.953, "update_time_ms": 8.36, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4151.5537109375, "policy_loss": 0.00024308195861522108, "vf_loss": 4151.55322265625, "vf_explained_var": 0.3201275169849396, "vf_preds": 115.34394836425781, "kl": 0.0054592564702034, "entropy": 0.055739350616931915, "entropy_coeff": 0.0, "advantages": 3.296330987723195e-06, "rewards": 1.8712904453277588}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2193000, "episodes_total": 2193, "training_iteration": 129, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_08-37-03", "timestamp": 1599295023, "time_this_iter_s": 2240.5376403331757, "time_total_s": 285629.74780774117, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc6c0268>", "on_episode_step": ".on_episode_step at 0x7f3afc6c0048>", "on_episode_end": ".on_episode_end at 0x7f3afc6c0d08>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc6c07b8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 285629.74780774117, "timesteps_since_restore": 2193000, "iterations_since_restore": 129, "perf": {"cpu_util_percent": 88.9158884362269, "ram_util_percent": 30.53885929175807}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144015.92021613938, "episode_reward_min": 140416.06022217526, "episode_reward_mean": 142386.7935800124, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 619.7305787470108}, "policy_reward_mean": {"av": 444.833620481778}, "custom_metrics": {"avg_speed_mean": 5.021303626917, "avg_speed_min": 4.973467590879922, "avg_speed_max": 5.068056917447317, "avg_speed_avs_mean": 5.022360333789197, "avg_speed_avs_min": 4.973076030684588, "avg_speed_avs_max": 5.070141429799276, "avg_headway_avs_mean": 7.005491463457761, "avg_headway_avs_min": 6.5413449946133895, "avg_headway_avs_max": 7.403901822350034, "std_headway_avs_mean": 6.6370219743816135, "std_headway_avs_min": 5.765621376636129, "std_headway_avs_max": 7.35516444389472, "avg_accel_avs_mean": 0.3471974702300922, "avg_accel_avs_min": 0.32340959715932593, "avg_accel_avs_max": 0.3683431608061374, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 29.207603381311255, "avg_mpg_per_veh_min": 27.915550211094285, "avg_mpg_per_veh_max": 30.339968866156013, "num_cars_mean": 933.56763, "num_cars_min": 924.582, "num_cars_max": 946.002, "avg_mpg_mean": 29.046918446162103, "avg_mpg_min": 27.77290129152728, "avg_mpg_max": 30.18602389187769}, "sampler_perf": {"mean_env_wait_ms": 1569.7142154137118, "mean_processing_ms": 428.60780649455324, "mean_inference_ms": 1.8594930444727147}, "off_policy_estimator": {}, "info": {"num_steps_trained": 157079424, "num_steps_sampled": 2210000, "sample_time_ms": 2051724.449, "load_time_ms": 477.516, "grad_time_ms": 185896.117, "update_time_ms": 8.415, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4181.3037109375, "policy_loss": 0.00020239177683833987, "vf_loss": 4181.3037109375, "vf_explained_var": 0.3453139364719391, "vf_preds": 121.36927032470703, "kl": 0.005130917299538851, "entropy": 0.005392695777118206, "entropy_coeff": 0.0, "advantages": 9.306665560870897e-06, "rewards": 1.8732904195785522}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2210000, "episodes_total": 2210, "training_iteration": 130, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_09-14-19", "timestamp": 1599297259, "time_this_iter_s": 2235.408141374588, "time_total_s": 287865.15594911575, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc3b0730>", "on_episode_step": ".on_episode_step at 0x7f3afc3b0ae8>", "on_episode_end": ".on_episode_end at 0x7f3afc3b0488>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc3b0950>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 287865.15594911575, "timesteps_since_restore": 2210000, "iterations_since_restore": 130, "perf": {"cpu_util_percent": 88.14855527638191, "ram_util_percent": 30.55957914572864}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144015.92021613938, "episode_reward_min": 140416.06022217526, "episode_reward_mean": 142413.81015879396, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 619.7305787470108}, "policy_reward_mean": {"av": 444.7929607058339}, "custom_metrics": {"avg_speed_mean": 5.0218106388271755, "avg_speed_min": 4.973467590879922, "avg_speed_max": 5.068056917447317, "avg_speed_avs_mean": 5.022985275651368, "avg_speed_avs_min": 4.973076030684588, "avg_speed_avs_max": 5.070141429799276, "avg_headway_avs_mean": 7.007231853038114, "avg_headway_avs_min": 6.5413449946133895, "avg_headway_avs_max": 7.403901822350034, "std_headway_avs_mean": 6.65088559314086, "std_headway_avs_min": 5.765621376636129, "std_headway_avs_max": 7.35516444389472, "avg_accel_avs_mean": 0.3473161525675288, "avg_accel_avs_min": 0.3238887817104322, "avg_accel_avs_max": 0.3683431608061374, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 29.231495305748016, "avg_mpg_per_veh_min": 27.915550211094285, "avg_mpg_per_veh_max": 30.339968866156013, "num_cars_mean": 933.5032700000002, "num_cars_min": 924.582, "num_cars_max": 946.002, "avg_mpg_mean": 29.07137164716896, "avg_mpg_min": 27.77290129152728, "avg_mpg_max": 30.18602389187769}, "sampler_perf": {"mean_env_wait_ms": 1569.7429173238568, "mean_processing_ms": 428.7799075002671, "mean_inference_ms": 1.8592168368855335}, "off_policy_estimator": {}, "info": {"num_steps_trained": 158327680, "num_steps_sampled": 2227000, "sample_time_ms": 2050219.579, "load_time_ms": 478.313, "grad_time_ms": 186023.031, "update_time_ms": 8.442, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4349.59521484375, "policy_loss": 0.0002757639158517122, "vf_loss": 4349.59521484375, "vf_explained_var": 0.3140256702899933, "vf_preds": 119.92232513427734, "kl": 0.00510264839977026, "entropy": 0.04747592657804489, "entropy_coeff": 0.0, "advantages": -6.987353572185384e-06, "rewards": 1.8792952299118042}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2227000, "episodes_total": 2227, "training_iteration": 131, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_09-51-27", "timestamp": 1599299487, "time_this_iter_s": 2228.1429665088654, "time_total_s": 290093.2989156246, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc4fc730>", "on_episode_step": ".on_episode_step at 0x7f3afc4fcf28>", "on_episode_end": ".on_episode_end at 0x7f3afc4fc158>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc4fcea0>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 290093.2989156246, "timesteps_since_restore": 2227000, "iterations_since_restore": 131, "perf": {"cpu_util_percent": 87.88168925307279, "ram_util_percent": 30.594673810274188}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144247.58658747099, "episode_reward_min": 140416.06022217526, "episode_reward_mean": 142462.83062882785, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 619.7305787470108}, "policy_reward_mean": {"av": 444.8488075841619}, "custom_metrics": {"avg_speed_mean": 5.021480419428375, "avg_speed_min": 4.97410426684115, "avg_speed_max": 5.068056917447317, "avg_speed_avs_mean": 5.022829634690072, "avg_speed_avs_min": 4.973076030684588, "avg_speed_avs_max": 5.070141429799276, "avg_headway_avs_mean": 7.019334497739882, "avg_headway_avs_min": 6.5413449946133895, "avg_headway_avs_max": 7.403901822350034, "std_headway_avs_mean": 6.6547526379950455, "std_headway_avs_min": 6.000483311284352, "std_headway_avs_max": 7.35516444389472, "avg_accel_avs_mean": 0.34736421011785845, "avg_accel_avs_min": 0.3266696024610969, "avg_accel_avs_max": 0.3683431608061374, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 29.257110369465657, "avg_mpg_per_veh_min": 27.915550211094285, "avg_mpg_per_veh_max": 30.494575778952974, "num_cars_mean": 933.54172, "num_cars_min": 924.582, "num_cars_max": 943.901, "avg_mpg_mean": 29.096178574744275, "avg_mpg_min": 27.77290129152728, "avg_mpg_max": 30.31206410850506}, "sampler_perf": {"mean_env_wait_ms": 1569.7638401065187, "mean_processing_ms": 428.9500601097921, "mean_inference_ms": 1.8589327322411713}, "off_policy_estimator": {}, "info": {"num_steps_trained": 159581312, "num_steps_sampled": 2244000, "sample_time_ms": 2051054.659, "load_time_ms": 473.994, "grad_time_ms": 186258.46, "update_time_ms": 8.531, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4594.24951171875, "policy_loss": 0.00028847678913734853, "vf_loss": 4594.25, "vf_explained_var": 0.2633330523967743, "vf_preds": 116.42530822753906, "kl": 0.005364942830055952, "entropy": 0.08346949517726898, "entropy_coeff": 0.0, "advantages": -5.1803367568936665e-06, "rewards": 1.8801374435424805}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2244000, "episodes_total": 2244, "training_iteration": 132, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_10-28-45", "timestamp": 1599301725, "time_this_iter_s": 2238.013192653656, "time_total_s": 292331.3121082783, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc704c80>", "on_episode_step": ".on_episode_step at 0x7f3afc704048>", "on_episode_end": ".on_episode_end at 0x7f3afc570620>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc570950>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 292331.3121082783, "timesteps_since_restore": 2244000, "iterations_since_restore": 132, "perf": {"cpu_util_percent": 87.5585947302384, "ram_util_percent": 30.567879548306145}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144615.20653479837, "episode_reward_min": 140416.06022217526, "episode_reward_mean": 142722.8899252829, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 618.4432884757961}, "policy_reward_mean": {"av": 445.54955803478566}, "custom_metrics": {"avg_speed_mean": 5.015551788105655, "avg_speed_min": 4.934923223318799, "avg_speed_max": 5.068056917447317, "avg_speed_avs_mean": 5.017123985238772, "avg_speed_avs_min": 4.936946550860656, "avg_speed_avs_max": 5.070141429799276, "avg_headway_avs_mean": 7.011465255285201, "avg_headway_avs_min": 6.381999745455852, "avg_headway_avs_max": 7.403901822350034, "std_headway_avs_mean": 6.5566189932157215, "std_headway_avs_min": 5.37750660204992, "std_headway_avs_max": 7.35516444389472, "avg_accel_avs_mean": 0.34340565097446907, "avg_accel_avs_min": 0.3135248615098786, "avg_accel_avs_max": 0.3683431608061374, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 29.450006544646868, "avg_mpg_per_veh_min": 27.915550211094285, "avg_mpg_per_veh_max": 30.84762604887364, "num_cars_mean": 935.2879999999998, "num_cars_min": 924.582, "num_cars_max": 952.458, "avg_mpg_mean": 29.288778270803306, "avg_mpg_min": 27.77290129152728, "avg_mpg_max": 30.689834239193665}, "sampler_perf": {"mean_env_wait_ms": 1569.7889267143685, "mean_processing_ms": 429.1175093530912, "mean_inference_ms": 1.8586295272350375}, "off_policy_estimator": {}, "info": {"num_steps_trained": 160839040, "num_steps_sampled": 2261000, "sample_time_ms": 2051795.245, "load_time_ms": 474.936, "grad_time_ms": 186373.653, "update_time_ms": 8.536, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4631.87158203125, "policy_loss": 0.00022603938123211265, "vf_loss": 4631.87158203125, "vf_explained_var": 0.20276999473571777, "vf_preds": 114.63584899902344, "kl": 0.005471016746014357, "entropy": 0.1043093279004097, "entropy_coeff": 0.0, "advantages": -2.590660642454168e-06, "rewards": 1.8808214664459229}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2261000, "episodes_total": 2261, "training_iteration": 133, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_11-06-17", "timestamp": 1599303977, "time_this_iter_s": 2252.553726196289, "time_total_s": 294583.86583447456, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc597bf8>", "on_episode_step": ".on_episode_step at 0x7f3afc597b70>", "on_episode_end": ".on_episode_end at 0x7f3afc597598>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc597488>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 294583.86583447456, "timesteps_since_restore": 2261000, "iterations_since_restore": 133, "perf": {"cpu_util_percent": 88.47297381546136, "ram_util_percent": 30.700748129675812}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144716.72064260097, "episode_reward_min": 141681.82954791948, "episode_reward_mean": 143205.7184780288, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 621.8123621127619}, "policy_reward_mean": {"av": 446.8057735422571}, "custom_metrics": {"avg_speed_mean": 5.008848426092719, "avg_speed_min": 4.934923223318799, "avg_speed_max": 5.068056917447317, "avg_speed_avs_mean": 5.010281708173476, "avg_speed_avs_min": 4.936946550860656, "avg_speed_avs_max": 5.069263147124388, "avg_headway_avs_mean": 7.014190732594775, "avg_headway_avs_min": 6.381999745455852, "avg_headway_avs_max": 7.392008734061879, "std_headway_avs_mean": 6.415491766379943, "std_headway_avs_min": 5.37750660204992, "std_headway_avs_max": 7.2647507498296955, "avg_accel_avs_mean": 0.33719977523156464, "avg_accel_avs_min": 0.3135248615098786, "avg_accel_avs_max": 0.35904549577965134, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 29.79751699882381, "avg_mpg_per_veh_min": 28.800316865442323, "avg_mpg_per_veh_max": 30.84762604887364, "num_cars_mean": 937.81598, "num_cars_min": 925.988, "num_cars_max": 952.458, "avg_mpg_mean": 29.633810313697776, "avg_mpg_min": 28.62653625938594, "avg_mpg_max": 30.689834239193665}, "sampler_perf": {"mean_env_wait_ms": 1569.8651192987675, "mean_processing_ms": 429.28427478752457, "mean_inference_ms": 1.8583660486615208}, "off_policy_estimator": {}, "info": {"num_steps_trained": 162094848, "num_steps_sampled": 2278000, "sample_time_ms": 2054138.324, "load_time_ms": 474.656, "grad_time_ms": 186250.817, "update_time_ms": 8.501, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4900.71337890625, "policy_loss": 0.00018460300634615123, "vf_loss": 4900.71337890625, "vf_explained_var": 0.19586829841136932, "vf_preds": 116.80484771728516, "kl": 0.005450818687677383, "entropy": 0.11490746587514877, "entropy_coeff": 0.0, "advantages": 2.4789085273368983e-06, "rewards": 1.8857080936431885}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2278000, "episodes_total": 2278, "training_iteration": 134, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_11-43-55", "timestamp": 1599306235, "time_this_iter_s": 2257.190518140793, "time_total_s": 296841.05635261536, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc75f0d0>", "on_episode_step": ".on_episode_step at 0x7f3afc75fe18>", "on_episode_end": ".on_episode_end at 0x7f3afc5977b8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc597f28>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 296841.05635261536, "timesteps_since_restore": 2278000, "iterations_since_restore": 134, "perf": {"cpu_util_percent": 88.34071539657855, "ram_util_percent": 30.62279937791602}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144716.72064260097, "episode_reward_min": 141681.82954791948, "episode_reward_mean": 143520.37437413694, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 621.8123621127619}, "policy_reward_mean": {"av": 447.8294257805071}, "custom_metrics": {"avg_speed_mean": 5.0056438731193955, "avg_speed_min": 4.934923223318799, "avg_speed_max": 5.068019300749835, "avg_speed_avs_mean": 5.006815089542084, "avg_speed_avs_min": 4.936946550860656, "avg_speed_avs_max": 5.069263147124388, "avg_headway_avs_mean": 7.136432911033766, "avg_headway_avs_min": 6.381999745455852, "avg_headway_avs_max": 7.886683576473862, "std_headway_avs_mean": 6.357525352794353, "std_headway_avs_min": 5.37750660204992, "std_headway_avs_max": 7.2647507498296955, "avg_accel_avs_mean": 0.3309401186204097, "avg_accel_avs_min": 0.3059147064735283, "avg_accel_avs_max": 0.356973077231546, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 30.081310608192986, "avg_mpg_per_veh_min": 28.800316865442323, "avg_mpg_per_veh_max": 31.277889370683514, "num_cars_mean": 939.39138, "num_cars_min": 926.918, "num_cars_max": 952.458, "avg_mpg_mean": 29.91498747387925, "avg_mpg_min": 28.62653625938594, "avg_mpg_max": 31.12836714414317}, "sampler_perf": {"mean_env_wait_ms": 1569.9685478285026, "mean_processing_ms": 429.44957720494693, "mean_inference_ms": 1.85815026888932}, "off_policy_estimator": {}, "info": {"num_steps_trained": 163346048, "num_steps_sampled": 2295000, "sample_time_ms": 2055667.856, "load_time_ms": 470.681, "grad_time_ms": 186189.287, "update_time_ms": 8.514, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5063.5859375, "policy_loss": 4.960423393640667e-05, "vf_loss": 5063.5859375, "vf_explained_var": 0.14503024518489838, "vf_preds": 116.45258331298828, "kl": 0.00623443815857172, "entropy": 0.11251803487539291, "entropy_coeff": 0.0, "advantages": 3.271462674092618e-06, "rewards": 1.8935457468032837}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2295000, "episodes_total": 2295, "training_iteration": 135, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_12-21-36", "timestamp": 1599308496, "time_this_iter_s": 2260.9366207122803, "time_total_s": 299101.99297332764, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc597a60>", "on_episode_step": ".on_episode_step at 0x7f3afc5e9e18>", "on_episode_end": ".on_episode_end at 0x7f3afc5e99d8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5e9ae8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 299101.99297332764, "timesteps_since_restore": 2295000, "iterations_since_restore": 135, "perf": {"cpu_util_percent": 89.06723602484472, "ram_util_percent": 30.637639751552793}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144716.72064260097, "episode_reward_min": 142262.10815814597, "episode_reward_mean": 143788.54869116674, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 621.8737568997894}, "policy_reward_mean": {"av": 448.6662153368908}, "custom_metrics": {"avg_speed_mean": 4.9987157358912695, "avg_speed_min": 4.934923223318799, "avg_speed_max": 5.049119385607065, "avg_speed_avs_mean": 4.9991369790797116, "avg_speed_avs_min": 4.936946550860656, "avg_speed_avs_max": 5.048752787148095, "avg_headway_avs_mean": 7.308880827468622, "avg_headway_avs_min": 6.381999745455852, "avg_headway_avs_max": 8.327421669481772, "std_headway_avs_mean": 6.317587369412771, "std_headway_avs_min": 5.37750660204992, "std_headway_avs_max": 7.033013569379829, "avg_accel_avs_mean": 0.32492558893683865, "avg_accel_avs_min": 0.3059147064735283, "avg_accel_avs_max": 0.3519747284310902, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 30.39460924735238, "avg_mpg_per_veh_min": 29.10217730924266, "avg_mpg_per_veh_max": 31.37924597647957, "num_cars_mean": 941.26059, "num_cars_min": 927.92, "num_cars_max": 952.458, "avg_mpg_mean": 30.228856000575277, "avg_mpg_min": 28.93798135956646, "avg_mpg_max": 31.25643464396483}, "sampler_perf": {"mean_env_wait_ms": 1570.1049509899033, "mean_processing_ms": 429.61303099495143, "mean_inference_ms": 1.8579521215660764}, "off_policy_estimator": {}, "info": {"num_steps_trained": 164594432, "num_steps_sampled": 2312000, "sample_time_ms": 2057647.58, "load_time_ms": 483.787, "grad_time_ms": 185986.1, "update_time_ms": 8.512, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4816.189453125, "policy_loss": 0.00022165571863297373, "vf_loss": 4816.18896484375, "vf_explained_var": 0.12320449203252792, "vf_preds": 117.97715759277344, "kl": 0.006271167658269405, "entropy": 0.11070210486650467, "entropy_coeff": 0.0, "advantages": 9.74187241808977e-06, "rewards": 1.8903237581253052}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2312000, "episodes_total": 2312, "training_iteration": 136, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_12-59-13", "timestamp": 1599310753, "time_this_iter_s": 2257.337792158127, "time_total_s": 301359.33076548576, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afff9b8c8>", "on_episode_step": ".on_episode_step at 0x7f3afff9b840>", "on_episode_end": ".on_episode_end at 0x7f3afff9b1e0>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afff9bae8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 301359.33076548576, "timesteps_since_restore": 2312000, "iterations_since_restore": 136, "perf": {"cpu_util_percent": 88.82746500777607, "ram_util_percent": 30.61216174183515}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144716.72064260097, "episode_reward_min": 142944.5441188036, "episode_reward_mean": 143942.89662503795, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 621.8737568997894}, "policy_reward_mean": {"av": 449.38620906321364}, "custom_metrics": {"avg_speed_mean": 4.994751884425941, "avg_speed_min": 4.934923223318799, "avg_speed_max": 5.043516094721612, "avg_speed_avs_mean": 4.994637443797553, "avg_speed_avs_min": 4.936946550860656, "avg_speed_avs_max": 5.034254698861614, "avg_headway_avs_mean": 7.500082922023678, "avg_headway_avs_min": 6.381999745455852, "avg_headway_avs_max": 8.589762532237454, "std_headway_avs_mean": 6.320097147896578, "std_headway_avs_min": 5.37750660204992, "std_headway_avs_max": 6.950824414461461, "avg_accel_avs_mean": 0.319214034815057, "avg_accel_avs_min": 0.3043839452494431, "avg_accel_avs_max": 0.3400358171869724, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 30.63261174123113, "avg_mpg_per_veh_min": 29.510656301531874, "avg_mpg_per_veh_max": 31.441195273600687, "num_cars_mean": 942.44783, "num_cars_min": 935.373, "num_cars_max": 952.458, "avg_mpg_mean": 30.46407269335448, "avg_mpg_min": 29.339048131601405, "avg_mpg_max": 31.25643464396483}, "sampler_perf": {"mean_env_wait_ms": 1570.2607815304177, "mean_processing_ms": 429.77345779714466, "mean_inference_ms": 1.8577772724800754}, "off_policy_estimator": {}, "info": {"num_steps_trained": 165840128, "num_steps_sampled": 2329000, "sample_time_ms": 2056363.327, "load_time_ms": 480.267, "grad_time_ms": 186026.928, "update_time_ms": 8.24, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4592.85693359375, "policy_loss": 0.0003400496207177639, "vf_loss": 4592.8564453125, "vf_explained_var": 0.12351036071777344, "vf_preds": 119.15833282470703, "kl": 0.005486589856445789, "entropy": 0.08306071907281876, "entropy_coeff": 0.0, "advantages": 2.073198402285925e-06, "rewards": 1.8953304290771484}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2329000, "episodes_total": 2329, "training_iteration": 137, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_13-36-43", "timestamp": 1599313003, "time_this_iter_s": 2249.5351243019104, "time_total_s": 303608.8658897877, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc597598>", "on_episode_step": ".on_episode_step at 0x7f3afc597a60>", "on_episode_end": ".on_episode_end at 0x7f3afc5e98c8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5e9488>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 303608.8658897877, "timesteps_since_restore": 2329000, "iterations_since_restore": 137, "perf": {"cpu_util_percent": 88.15930087390763, "ram_util_percent": 30.507771535580527}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144716.72064260097, "episode_reward_min": 142944.5441188036, "episode_reward_mean": 143962.98856341318, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 624.0735797900926}, "policy_reward_mean": {"av": 449.88433926066625}, "custom_metrics": {"avg_speed_mean": 4.997532231838868, "avg_speed_min": 4.934923223318799, "avg_speed_max": 5.044891630625116, "avg_speed_avs_mean": 4.996727503325225, "avg_speed_avs_min": 4.936946550860656, "avg_speed_avs_max": 5.043414752309643, "avg_headway_avs_mean": 7.748396858561164, "avg_headway_avs_min": 6.381999745455852, "avg_headway_avs_max": 8.669382072162312, "std_headway_avs_mean": 6.427281973507504, "std_headway_avs_min": 5.37750660204992, "std_headway_avs_max": 6.9804966655911995, "avg_accel_avs_mean": 0.3159586366685865, "avg_accel_avs_min": 0.3043839452494431, "avg_accel_avs_max": 0.33098042403828576, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 30.80450561287952, "avg_mpg_per_veh_min": 30.08959488344761, "avg_mpg_per_veh_max": 31.441195273600687, "num_cars_mean": 942.24655, "num_cars_min": 935.373, "num_cars_max": 952.458, "avg_mpg_mean": 30.634568888665314, "avg_mpg_min": 29.929567336922887, "avg_mpg_max": 31.25643464396483}, "sampler_perf": {"mean_env_wait_ms": 1570.4130392091831, "mean_processing_ms": 429.92916040154995, "mean_inference_ms": 1.8576015793683955}, "off_policy_estimator": {}, "info": {"num_steps_trained": 167081856, "num_steps_sampled": 2346000, "sample_time_ms": 2058989.675, "load_time_ms": 480.193, "grad_time_ms": 186562.874, "update_time_ms": 8.792, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4680.14990234375, "policy_loss": 0.0003898880968336016, "vf_loss": 4680.14990234375, "vf_explained_var": 0.13278068602085114, "vf_preds": 123.07685852050781, "kl": 0.004861909430474043, "entropy": 0.08405213057994843, "entropy_coeff": 0.0, "advantages": 4.825779797101859e-06, "rewards": 1.900884747505188}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2346000, "episodes_total": 2346, "training_iteration": 138, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_14-14-08", "timestamp": 1599315248, "time_this_iter_s": 2244.7629396915436, "time_total_s": 305853.6288294792, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc627488>", "on_episode_step": ".on_episode_step at 0x7f3afc704048>", "on_episode_end": ".on_episode_end at 0x7f3afc704c80>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc6c0048>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 305853.6288294792, "timesteps_since_restore": 2346000, "iterations_since_restore": 138, "perf": {"cpu_util_percent": 87.68151391929933, "ram_util_percent": 30.533687832342824}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144716.72064260097, "episode_reward_min": 142765.8055096284, "episode_reward_mean": 143824.2515343349, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 624.0735797900926}, "policy_reward_mean": {"av": 450.0696317885058}, "custom_metrics": {"avg_speed_mean": 4.99901235134405, "avg_speed_min": 4.9529307221636, "avg_speed_max": 5.044891630625116, "avg_speed_avs_mean": 4.997276580482267, "avg_speed_avs_min": 4.953260676376714, "avg_speed_avs_max": 5.043414752309643, "avg_headway_avs_mean": 8.032897429288708, "avg_headway_avs_min": 6.810031133745214, "avg_headway_avs_max": 9.089083698747203, "std_headway_avs_mean": 6.603202507951737, "std_headway_avs_min": 5.884635763979586, "std_headway_avs_max": 7.3347627524432175, "avg_accel_avs_mean": 0.31401735255743635, "avg_accel_avs_min": 0.3043839452494431, "avg_accel_avs_max": 0.33098042403828576, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 30.894955399309794, "avg_mpg_per_veh_min": 30.141725376809223, "avg_mpg_per_veh_max": 31.441195273600687, "num_cars_mean": 941.4205200000001, "num_cars_min": 935.24, "num_cars_max": 947.429, "avg_mpg_mean": 30.725604039466766, "avg_mpg_min": 29.974479734814143, "avg_mpg_max": 31.25643464396483}, "sampler_perf": {"mean_env_wait_ms": 1570.5605974333087, "mean_processing_ms": 430.0822865153412, "mean_inference_ms": 1.857429249469576}, "off_policy_estimator": {}, "info": {"num_steps_trained": 168323712, "num_steps_sampled": 2363000, "sample_time_ms": 2059760.927, "load_time_ms": 480.645, "grad_time_ms": 186367.62, "update_time_ms": 8.508, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4794.26708984375, "policy_loss": 0.00030564810731448233, "vf_loss": 4794.2666015625, "vf_explained_var": 0.11740854382514954, "vf_preds": 123.58158111572266, "kl": 0.005603594705462456, "entropy": 0.07516627013683319, "entropy_coeff": 0.0, "advantages": -1.9524975414242363e-06, "rewards": 1.8937523365020752}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2363000, "episodes_total": 2363, "training_iteration": 139, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_14-51-34", "timestamp": 1599317494, "time_this_iter_s": 2246.278062582016, "time_total_s": 308099.90689206123, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5a4e18>", "on_episode_step": ".on_episode_step at 0x7f3afc5a4f28>", "on_episode_end": ".on_episode_end at 0x7f3afc5a4378>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5a4ea0>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 308099.90689206123, "timesteps_since_restore": 2363000, "iterations_since_restore": 139, "perf": {"cpu_util_percent": 88.8039074710847, "ram_util_percent": 30.50965926852141}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144698.38580159793, "episode_reward_min": 141436.59261709324, "episode_reward_mean": 143527.1116738144, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 624.0735797900926}, "policy_reward_mean": {"av": 450.1257971329561}, "custom_metrics": {"avg_speed_mean": 4.994008376072365, "avg_speed_min": 4.90313367156459, "avg_speed_max": 5.044891630625116, "avg_speed_avs_mean": 4.99103916536416, "avg_speed_avs_min": 4.896160282546108, "avg_speed_avs_max": 5.043414752309643, "avg_headway_avs_mean": 8.34202982748345, "avg_headway_avs_min": 7.475101381141319, "avg_headway_avs_max": 9.447092971228551, "std_headway_avs_mean": 6.7676911414064636, "std_headway_avs_min": 6.084865501184294, "std_headway_avs_max": 7.633497504164503, "avg_accel_avs_mean": 0.3114568577029599, "avg_accel_avs_min": 0.3038009466206521, "avg_accel_avs_max": 0.32009820943925626, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 30.977687780589225, "avg_mpg_per_veh_min": 30.360644713630855, "avg_mpg_per_veh_max": 31.441195273600687, "num_cars_mean": 940.81555, "num_cars_min": 933.832, "num_cars_max": 947.263, "avg_mpg_mean": 30.809770526547137, "avg_mpg_min": 30.18157123670491, "avg_mpg_max": 31.25643464396483}, "sampler_perf": {"mean_env_wait_ms": 1570.6925478775602, "mean_processing_ms": 430.22969701693944, "mean_inference_ms": 1.8572326279248128}, "off_policy_estimator": {}, "info": {"num_steps_trained": 169565952, "num_steps_sampled": 2380000, "sample_time_ms": 2061079.448, "load_time_ms": 482.675, "grad_time_ms": 186615.764, "update_time_ms": 8.476, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4779.99609375, "policy_loss": 0.000469744554720819, "vf_loss": 4779.99560546875, "vf_explained_var": 0.10907024890184402, "vf_preds": 125.07140350341797, "kl": 0.005436517298221588, "entropy": 0.06298753619194031, "entropy_coeff": 0.0, "advantages": -7.083580385369714e-06, "rewards": 1.8824901580810547}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2380000, "episodes_total": 2380, "training_iteration": 140, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_15-29-05", "timestamp": 1599319745, "time_this_iter_s": 2251.078336954117, "time_total_s": 310350.98522901535, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3b003b39d8>", "on_episode_step": ".on_episode_step at 0x7f3b003b3d90>", "on_episode_end": ".on_episode_end at 0x7f3b003b3950>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3b003b3f28>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 310350.98522901535, "timesteps_since_restore": 2380000, "iterations_since_restore": 140, "perf": {"cpu_util_percent": 87.9172435297786, "ram_util_percent": 30.547801683816658}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144278.22840401236, "episode_reward_min": 140109.2314259273, "episode_reward_mean": 142963.47178482285, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 624.0735797900926}, "policy_reward_mean": {"av": 449.6555066516413}, "custom_metrics": {"avg_speed_mean": 4.978365419183203, "avg_speed_min": 4.871196661568082, "avg_speed_max": 5.044891630625116, "avg_speed_avs_mean": 4.97416050658504, "avg_speed_avs_min": 4.858767204397261, "avg_speed_avs_max": 5.043414752309643, "avg_headway_avs_mean": 8.576462589503414, "avg_headway_avs_min": 7.74451092850852, "avg_headway_avs_max": 9.447092971228551, "std_headway_avs_mean": 6.930615570739633, "std_headway_avs_min": 6.25383956382914, "std_headway_avs_max": 7.633497504164503, "avg_accel_avs_mean": 0.3109516234561693, "avg_accel_avs_min": 0.3030611242467128, "avg_accel_avs_max": 0.32009820943925626, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 30.928556380296627, "avg_mpg_per_veh_min": 30.357010927555304, "avg_mpg_per_veh_max": 31.441195273600687, "num_cars_mean": 940.83826, "num_cars_min": 933.832, "num_cars_max": 946.698, "avg_mpg_mean": 30.76624488878315, "avg_mpg_min": 30.233093768244878, "avg_mpg_max": 31.25643464396483}, "sampler_perf": {"mean_env_wait_ms": 1570.8032698307754, "mean_processing_ms": 430.37441844221377, "mean_inference_ms": 1.8569811315963134}, "off_policy_estimator": {}, "info": {"num_steps_trained": 170812160, "num_steps_sampled": 2397000, "sample_time_ms": 2063122.284, "load_time_ms": 481.219, "grad_time_ms": 185814.497, "update_time_ms": 8.114, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4792.57275390625, "policy_loss": 0.0004501582880038768, "vf_loss": 4792.5712890625, "vf_explained_var": 0.1050284132361412, "vf_preds": 127.74600982666016, "kl": 0.005278749391436577, "entropy": 0.041374173015356064, "entropy_coeff": 0.0, "advantages": 9.420316473551793e-07, "rewards": 1.856900691986084}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2397000, "episodes_total": 2397, "training_iteration": 141, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_16-06-26", "timestamp": 1599321986, "time_this_iter_s": 2240.542020559311, "time_total_s": 312591.52724957466, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc71e950>", "on_episode_step": ".on_episode_step at 0x7f3afc71e840>", "on_episode_end": ".on_episode_end at 0x7f3afc71e048>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc71e2f0>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 312591.52724957466, "timesteps_since_restore": 2397000, "iterations_since_restore": 141, "perf": {"cpu_util_percent": 88.15314948292072, "ram_util_percent": 30.617800062676274}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144278.22840401236, "episode_reward_min": 140109.2314259273, "episode_reward_mean": 142609.97075577092, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 624.0735797900926}, "policy_reward_mean": {"av": 449.56172610734166}, "custom_metrics": {"avg_speed_mean": 4.97143488385685, "avg_speed_min": 4.871196661568082, "avg_speed_max": 5.044891630625116, "avg_speed_avs_mean": 4.966044439666087, "avg_speed_avs_min": 4.858767204397261, "avg_speed_avs_max": 5.043414752309643, "avg_headway_avs_mean": 8.787733444482637, "avg_headway_avs_min": 7.74451092850852, "avg_headway_avs_max": 9.933715091521695, "std_headway_avs_mean": 7.069755025254279, "std_headway_avs_min": 6.25383956382914, "std_headway_avs_max": 7.7176278408781, "avg_accel_avs_mean": 0.3090462308565049, "avg_accel_avs_min": 0.29835014857557807, "avg_accel_avs_max": 0.3172880321745249, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 30.927437911928223, "avg_mpg_per_veh_min": 30.357010927555304, "avg_mpg_per_veh_max": 31.441195273600687, "num_cars_mean": 940.3049300000001, "num_cars_min": 932.336, "num_cars_max": 946.698, "avg_mpg_mean": 30.766224637192707, "avg_mpg_min": 30.233093768244878, "avg_mpg_max": 31.25362627486716}, "sampler_perf": {"mean_env_wait_ms": 1570.9110170097952, "mean_processing_ms": 430.5140493365399, "mean_inference_ms": 1.8567398954016556}, "off_policy_estimator": {}, "info": {"num_steps_trained": 172052736, "num_steps_sampled": 2414000, "sample_time_ms": 2064553.832, "load_time_ms": 481.756, "grad_time_ms": 185120.922, "update_time_ms": 7.928, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4869.3935546875, "policy_loss": 0.000537359737791121, "vf_loss": 4869.39306640625, "vf_explained_var": 0.10385353863239288, "vf_preds": 129.59249877929688, "kl": 0.005299726035445929, "entropy": 0.003025328740477562, "entropy_coeff": 0.0, "advantages": 1.9031222109333612e-05, "rewards": 1.8746188879013062}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2414000, "episodes_total": 2414, "training_iteration": 142, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_16-43-51", "timestamp": 1599324231, "time_this_iter_s": 2245.409990310669, "time_total_s": 314836.93723988533, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc541048>", "on_episode_step": ".on_episode_step at 0x7f3afc5410d0>", "on_episode_end": ".on_episode_end at 0x7f3afc5416a8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc541d90>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 314836.93723988533, "timesteps_since_restore": 2414000, "iterations_since_restore": 142, "perf": {"cpu_util_percent": 87.91103814884303, "ram_util_percent": 30.607160725453408}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144223.02522442662, "episode_reward_min": 139633.2981345337, "episode_reward_mean": 142016.02073933207, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 623.4089996273365}, "policy_reward_mean": {"av": 448.9773347011857}, "custom_metrics": {"avg_speed_mean": 4.953744194075084, "avg_speed_min": 4.846487020142548, "avg_speed_max": 5.044891630625116, "avg_speed_avs_mean": 4.946617139226501, "avg_speed_avs_min": 4.836971378209115, "avg_speed_avs_max": 5.043414752309643, "avg_headway_avs_mean": 8.975997306782492, "avg_headway_avs_min": 8.24277272062747, "avg_headway_avs_max": 9.951022075844532, "std_headway_avs_mean": 7.191641738633632, "std_headway_avs_min": 6.644379036326969, "std_headway_avs_max": 7.78134399079317, "avg_accel_avs_mean": 0.30863129699715464, "avg_accel_avs_min": 0.29835014857557807, "avg_accel_avs_max": 0.3172880321745249, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 30.850912806506845, "avg_mpg_per_veh_min": 30.252897443249886, "avg_mpg_per_veh_max": 31.339513529790125, "num_cars_mean": 940.5364399999999, "num_cars_min": 932.336, "num_cars_max": 948.054, "avg_mpg_mean": 30.69138014481662, "avg_mpg_min": 30.087813347320147, "avg_mpg_max": 31.19294562745633}, "sampler_perf": {"mean_env_wait_ms": 1571.0490103872157, "mean_processing_ms": 430.6485605447278, "mean_inference_ms": 1.8565111624627244}, "off_policy_estimator": {}, "info": {"num_steps_trained": 173298816, "num_steps_sampled": 2431000, "sample_time_ms": 2065818.754, "load_time_ms": 481.676, "grad_time_ms": 184332.658, "update_time_ms": 7.899, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4941.76416015625, "policy_loss": 0.0004484277160372585, "vf_loss": 4941.763671875, "vf_explained_var": 0.10083509981632233, "vf_preds": 131.95816040039062, "kl": 0.0055964733473956585, "entropy": -0.013764855451881886, "entropy_coeff": 0.0, "advantages": 4.012607860204298e-06, "rewards": 1.8487557172775269}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2431000, "episodes_total": 2431, "training_iteration": 143, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_17-21-29", "timestamp": 1599326489, "time_this_iter_s": 2257.3634769916534, "time_total_s": 317094.300716877, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3b003b3ae8>", "on_episode_step": ".on_episode_step at 0x7f3afc7041e0>", "on_episode_end": ".on_episode_end at 0x7f3afc704950>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc71e048>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 317094.300716877, "timesteps_since_restore": 2431000, "iterations_since_restore": 143, "perf": {"cpu_util_percent": 87.83751166407464, "ram_util_percent": 30.605723172628302}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 143561.400923005, "episode_reward_min": 139633.2981345337, "episode_reward_mean": 141579.52278449704, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 623.4089996273365}, "policy_reward_mean": {"av": 448.57589121252465}, "custom_metrics": {"avg_speed_mean": 4.938504821647126, "avg_speed_min": 4.846487020142548, "avg_speed_max": 5.040480948429869, "avg_speed_avs_mean": 4.930113928118305, "avg_speed_avs_min": 4.836971378209115, "avg_speed_avs_max": 5.037751487858817, "avg_headway_avs_mean": 9.120332864859813, "avg_headway_avs_min": 8.245141268423097, "avg_headway_avs_max": 9.951022075844532, "std_headway_avs_mean": 7.281876873272079, "std_headway_avs_min": 6.730911949198565, "std_headway_avs_max": 7.78134399079317, "avg_accel_avs_mean": 0.3064763156118054, "avg_accel_avs_min": 0.29401016573747196, "avg_accel_avs_max": 0.31647775316680443, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 30.81626620859436, "avg_mpg_per_veh_min": 30.252897443249886, "avg_mpg_per_veh_max": 31.339513529790125, "num_cars_mean": 940.94172, "num_cars_min": 932.336, "num_cars_max": 948.183, "avg_mpg_mean": 30.659421274940442, "avg_mpg_min": 30.087813347320147, "avg_mpg_max": 31.19294562745633}, "sampler_perf": {"mean_env_wait_ms": 1571.2631144281634, "mean_processing_ms": 430.759471716715, "mean_inference_ms": 1.8563631848734519}, "off_policy_estimator": {}, "info": {"num_steps_trained": 174541952, "num_steps_sampled": 2448000, "sample_time_ms": 2068642.475, "load_time_ms": 485.249, "grad_time_ms": 187001.309, "update_time_ms": 8.107, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5091.892578125, "policy_loss": 0.000392733869375661, "vf_loss": 5091.89208984375, "vf_explained_var": 0.09682504087686539, "vf_preds": 134.02011108398438, "kl": 0.00566187733784318, "entropy": -0.012251297943294048, "entropy_coeff": 0.0, "advantages": -1.2212201454531169e-06, "rewards": 1.8636261224746704}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2448000, "episodes_total": 2448, "training_iteration": 144, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_18-00-01", "timestamp": 1599328801, "time_this_iter_s": 2312.171987771988, "time_total_s": 319406.472704649, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc4fc6a8>", "on_episode_step": ".on_episode_step at 0x7f3afc4fcea0>", "on_episode_end": ".on_episode_end at 0x7f3afc4fc8c8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc4fcd90>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 319406.472704649, "timesteps_since_restore": 2448000, "iterations_since_restore": 144, "perf": {"cpu_util_percent": 85.48293349529305, "ram_util_percent": 30.517916793197696}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 143184.1464482845, "episode_reward_min": 139633.2981345337, "episode_reward_mean": 141212.2418926171, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 623.4089996273365}, "policy_reward_mean": {"av": 448.207458555885}, "custom_metrics": {"avg_speed_mean": 4.925057907587396, "avg_speed_min": 4.846487020142548, "avg_speed_max": 5.040480948429869, "avg_speed_avs_mean": 4.915317122443978, "avg_speed_avs_min": 4.836971378209115, "avg_speed_avs_max": 5.037751487858817, "avg_headway_avs_mean": 9.225436839598133, "avg_headway_avs_min": 8.403783512589795, "avg_headway_avs_max": 9.951022075844532, "std_headway_avs_mean": 7.338061180419365, "std_headway_avs_min": 6.730911949198565, "std_headway_avs_max": 7.78134399079317, "avg_accel_avs_mean": 0.3032305236781644, "avg_accel_avs_min": 0.28968822926627946, "avg_accel_avs_max": 0.313813762723183, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 30.791547263529427, "avg_mpg_per_veh_min": 30.252897443249886, "avg_mpg_per_veh_max": 31.339513529790125, "num_cars_mean": 941.4131600000001, "num_cars_min": 932.336, "num_cars_max": 948.183, "avg_mpg_mean": 30.636952750202703, "avg_mpg_min": 30.087813347320147, "avg_mpg_max": 31.19294562745633}, "sampler_perf": {"mean_env_wait_ms": 1571.5909030920168, "mean_processing_ms": 430.8199395215768, "mean_inference_ms": 1.8563546136326445}, "off_policy_estimator": {}, "info": {"num_steps_trained": 175786368, "num_steps_sampled": 2465000, "sample_time_ms": 2072730.867, "load_time_ms": 482.998, "grad_time_ms": 189508.875, "update_time_ms": 7.91, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5177.3603515625, "policy_loss": 0.0005586522165685892, "vf_loss": 5177.359375, "vf_explained_var": 0.09720426052808762, "vf_preds": 134.9719696044922, "kl": 0.005569724831730127, "entropy": -0.05411764979362488, "entropy_coeff": 0.0, "advantages": -1.7848409470389015e-06, "rewards": 1.8595904111862183}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2465000, "episodes_total": 2465, "training_iteration": 145, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_18-38-48", "timestamp": 1599331128, "time_this_iter_s": 2326.873541355133, "time_total_s": 321733.3462460041, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3b003b3ae8>", "on_episode_step": ".on_episode_step at 0x7f3afc71eb70>", "on_episode_end": ".on_episode_end at 0x7f3afc71e9d8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc71e730>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 321733.3462460041, "timesteps_since_restore": 2465000, "iterations_since_restore": 145, "perf": {"cpu_util_percent": 86.99604707302355, "ram_util_percent": 30.64164152082076}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 142230.23799192172, "episode_reward_min": 139633.2981345337, "episode_reward_mean": 140932.31851337207, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 623.4089996273365}, "policy_reward_mean": {"av": 447.8449220292099}, "custom_metrics": {"avg_speed_mean": 4.913099468593333, "avg_speed_min": 4.846487020142548, "avg_speed_max": 4.991211190036703, "avg_speed_avs_mean": 4.90267343665496, "avg_speed_avs_min": 4.836971378209115, "avg_speed_avs_max": 4.980776951608459, "avg_headway_avs_mean": 9.27251650349275, "avg_headway_avs_min": 8.731708103659795, "avg_headway_avs_max": 9.951022075844532, "std_headway_avs_mean": 7.39151508298662, "std_headway_avs_min": 7.024870579478626, "std_headway_avs_max": 7.854323079831666, "avg_accel_avs_mean": 0.29920060757167466, "avg_accel_avs_min": 0.2805035212659777, "avg_accel_avs_max": 0.3120318024802629, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 30.761879504338452, "avg_mpg_per_veh_min": 30.252897443249886, "avg_mpg_per_veh_max": 31.201905323593344, "num_cars_mean": 942.02394, "num_cars_min": 932.336, "num_cars_max": 950.337, "avg_mpg_mean": 30.609878176199274, "avg_mpg_min": 30.087813347320147, "avg_mpg_max": 31.05826692074988}, "sampler_perf": {"mean_env_wait_ms": 1572.0550408801462, "mean_processing_ms": 430.8103331505969, "mean_inference_ms": 1.8565280454679098}, "off_policy_estimator": {}, "info": {"num_steps_trained": 177033600, "num_steps_sampled": 2482000, "sample_time_ms": 2077410.359, "load_time_ms": 470.036, "grad_time_ms": 191608.79, "update_time_ms": 7.957, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5181.91943359375, "policy_loss": 0.0005367588018998504, "vf_loss": 5181.9189453125, "vf_explained_var": 0.0968867689371109, "vf_preds": 136.0209197998047, "kl": 0.005526020657271147, "entropy": -0.09239102900028229, "entropy_coeff": 0.0, "advantages": -9.400846465723589e-06, "rewards": 1.8514959812164307}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2482000, "episodes_total": 2482, "training_iteration": 146, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_19-17-33", "timestamp": 1599333453, "time_this_iter_s": 2324.9917187690735, "time_total_s": 324058.3379647732, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc3b0598>", "on_episode_step": ".on_episode_step at 0x7f3afc3b01e0>", "on_episode_end": ".on_episode_end at 0x7f3afc3b0158>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc3b0840>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 324058.3379647732, "timesteps_since_restore": 2482000, "iterations_since_restore": 146, "perf": {"cpu_util_percent": 87.9094533373603, "ram_util_percent": 30.616430081546362}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 142230.23799192172, "episode_reward_min": 139633.2981345337, "episode_reward_mean": 140963.54356887788, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 619.6324028151922}, "policy_reward_mean": {"av": 448.0295698721605}, "custom_metrics": {"avg_speed_mean": 4.914378496203173, "avg_speed_min": 4.846487020142548, "avg_speed_max": 4.991211190036703, "avg_speed_avs_mean": 4.90342998544573, "avg_speed_avs_min": 4.836971378209115, "avg_speed_avs_max": 4.980776951608459, "avg_headway_avs_mean": 9.303096614246135, "avg_headway_avs_min": 8.731708103659795, "avg_headway_avs_max": 9.951022075844532, "std_headway_avs_mean": 7.437990120405034, "std_headway_avs_min": 7.024870579478626, "std_headway_avs_max": 7.854323079831666, "avg_accel_avs_mean": 0.2941309288639603, "avg_accel_avs_min": 0.27252718016460176, "avg_accel_avs_max": 0.3115154303916979, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 30.815447591857335, "avg_mpg_per_veh_min": 30.252897443249886, "avg_mpg_per_veh_max": 31.58412186466864, "num_cars_mean": 941.88578, "num_cars_min": 932.336, "num_cars_max": 950.337, "avg_mpg_mean": 30.661510307849117, "avg_mpg_min": 30.087813347320147, "avg_mpg_max": 31.46066607872185}, "sampler_perf": {"mean_env_wait_ms": 1572.6513452963732, "mean_processing_ms": 430.6879616728999, "mean_inference_ms": 1.8568703000934599}, "off_policy_estimator": {}, "info": {"num_steps_trained": 178278656, "num_steps_sampled": 2499000, "sample_time_ms": 2079252.843, "load_time_ms": 468.54, "grad_time_ms": 191795.097, "update_time_ms": 8.08, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5192.84619140625, "policy_loss": 0.00045137552660889924, "vf_loss": 5192.845703125, "vf_explained_var": 0.10014002025127411, "vf_preds": 133.23464965820312, "kl": 0.005949060898274183, "entropy": -0.1290465146303177, "entropy_coeff": 0.0, "advantages": 1.2583584521053126e-06, "rewards": 1.8621872663497925}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2499000, "episodes_total": 2499, "training_iteration": 147, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_19-55-23", "timestamp": 1599335723, "time_this_iter_s": 2269.804152727127, "time_total_s": 326328.1421175003, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3b73421268>", "on_episode_step": ".on_episode_step at 0x7f3b73421378>", "on_episode_end": ".on_episode_end at 0x7f3b734212f0>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3b734211e0>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 326328.1421175003, "timesteps_since_restore": 2499000, "iterations_since_restore": 147, "perf": {"cpu_util_percent": 88.8206928549335, "ram_util_percent": 30.610856789359726}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 142201.0802773975, "episode_reward_min": 139633.2981345337, "episode_reward_mean": 140989.43943721513, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 622.9321215790187}, "policy_reward_mean": {"av": 448.1403624716798}, "custom_metrics": {"avg_speed_mean": 4.913168461683478, "avg_speed_min": 4.846487020142548, "avg_speed_max": 4.975633301265602, "avg_speed_avs_mean": 4.902149509100465, "avg_speed_avs_min": 4.836971378209115, "avg_speed_avs_max": 4.961745075682993, "avg_headway_avs_mean": 9.291278841340496, "avg_headway_avs_min": 8.725614800592517, "avg_headway_avs_max": 9.890849512567058, "std_headway_avs_mean": 7.473363409291856, "std_headway_avs_min": 7.024870579478626, "std_headway_avs_max": 7.914646173168708, "avg_accel_avs_mean": 0.2896591944415377, "avg_accel_avs_min": 0.27252718016460176, "avg_accel_avs_max": 0.3115154303916979, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 30.855640883547384, "avg_mpg_per_veh_min": 30.29605245589893, "avg_mpg_per_veh_max": 31.58412186466864, "num_cars_mean": 942.08864, "num_cars_min": 934.023, "num_cars_max": 950.337, "avg_mpg_mean": 30.702122137437158, "avg_mpg_min": 30.125661246814012, "avg_mpg_max": 31.46066607872185}, "sampler_perf": {"mean_env_wait_ms": 1573.34930249439, "mean_processing_ms": 430.4580333710331, "mean_inference_ms": 1.8573402991876156}, "off_policy_estimator": {}, "info": {"num_steps_trained": 179520128, "num_steps_sampled": 2516000, "sample_time_ms": 2080953.085, "load_time_ms": 468.914, "grad_time_ms": 191080.352, "update_time_ms": 7.757, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5207.6796875, "policy_loss": 0.00045925809536129236, "vf_loss": 5207.67919921875, "vf_explained_var": 0.10154029726982117, "vf_preds": 132.01654052734375, "kl": 0.0058572120033204556, "entropy": -0.12297505885362625, "entropy_coeff": 0.0, "advantages": 1.3113501154293772e-05, "rewards": 1.8723015785217285}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2516000, "episodes_total": 2516, "training_iteration": 148, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_20-32-57", "timestamp": 1599337977, "time_this_iter_s": 2254.6363151073456, "time_total_s": 328582.77843260765, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afff9b488>", "on_episode_step": ".on_episode_step at 0x7f3afff9b840>", "on_episode_end": ".on_episode_end at 0x7f3afff9b1e0>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afff9b8c8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 328582.77843260765, "timesteps_since_restore": 2516000, "iterations_since_restore": 148, "perf": {"cpu_util_percent": 88.47079701120798, "ram_util_percent": 30.63169364881693}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 142757.23082535682, "episode_reward_min": 139936.87533468593, "episode_reward_mean": 141287.6097645559, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 622.9321215790187}, "policy_reward_mean": {"av": 448.5748159016921}, "custom_metrics": {"avg_speed_mean": 4.923370570506014, "avg_speed_min": 4.85674125094646, "avg_speed_max": 4.980737718083969, "avg_speed_avs_mean": 4.913261398856847, "avg_speed_avs_min": 4.842026601171827, "avg_speed_avs_max": 4.975396310093398, "avg_headway_avs_mean": 9.251519474506454, "avg_headway_avs_min": 8.725614800592517, "avg_headway_avs_max": 9.890849512567058, "std_headway_avs_mean": 7.478277303325115, "std_headway_avs_min": 7.024870579478626, "std_headway_avs_max": 7.914646173168708, "avg_accel_avs_mean": 0.2863968532967065, "avg_accel_avs_min": 0.27252718016460176, "avg_accel_avs_max": 0.3041588394016377, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 30.944338524991345, "avg_mpg_per_veh_min": 30.420018137666244, "avg_mpg_per_veh_max": 31.58412186466864, "num_cars_mean": 941.6663400000001, "num_cars_min": 934.023, "num_cars_max": 950.337, "avg_mpg_mean": 30.791869823697503, "avg_mpg_min": 30.277303737481972, "avg_mpg_max": 31.46066607872185}, "sampler_perf": {"mean_env_wait_ms": 1574.07369972057, "mean_processing_ms": 430.15175186864815, "mean_inference_ms": 1.8578849297178934}, "off_policy_estimator": {}, "info": {"num_steps_trained": 180762624, "num_steps_sampled": 2533000, "sample_time_ms": 2080274.785, "load_time_ms": 467.507, "grad_time_ms": 190967.517, "update_time_ms": 7.861, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5240.2109375, "policy_loss": 0.000377333490177989, "vf_loss": 5240.2099609375, "vf_explained_var": 0.10199657827615738, "vf_preds": 134.14756774902344, "kl": 0.005574975628405809, "entropy": -0.0611678846180439, "entropy_coeff": 0.0, "advantages": -2.9734255804214627e-06, "rewards": 1.879772663116455}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2533000, "episodes_total": 2533, "training_iteration": 149, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_21-10-16", "timestamp": 1599340216, "time_this_iter_s": 2238.3661777973175, "time_total_s": 330821.14461040497, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc3b0048>", "on_episode_step": ".on_episode_step at 0x7f3afc3b00d0>", "on_episode_end": ".on_episode_end at 0x7f3afc3b0268>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc3b02f0>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 330821.14461040497, "timesteps_since_restore": 2533000, "iterations_since_restore": 149, "perf": {"cpu_util_percent": 87.32685069008782, "ram_util_percent": 30.544134253450437}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 143272.81171143663, "episode_reward_min": 139936.87533468593, "episode_reward_mean": 141607.08873573784, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 622.9321215790187}, "policy_reward_mean": {"av": 449.0616120242845}, "custom_metrics": {"avg_speed_mean": 4.931350521556677, "avg_speed_min": 4.85674125094646, "avg_speed_max": 5.007542714737067, "avg_speed_avs_mean": 4.921967140695844, "avg_speed_avs_min": 4.842026601171827, "avg_speed_avs_max": 5.006035977378459, "avg_headway_avs_mean": 9.176966318634221, "avg_headway_avs_min": 8.607204395313987, "avg_headway_avs_max": 9.890849512567058, "std_headway_avs_mean": 7.439020501369429, "std_headway_avs_min": 6.987246280232479, "std_headway_avs_max": 7.914646173168708, "avg_accel_avs_mean": 0.2825727108072408, "avg_accel_avs_min": 0.27252718016460176, "avg_accel_avs_max": 0.29550355583965854, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.042207637523624, "avg_mpg_per_veh_min": 30.420018137666244, "avg_mpg_per_veh_max": 31.818847141882628, "num_cars_mean": 941.66355, "num_cars_min": 935.687, "num_cars_max": 950.337, "avg_mpg_mean": 30.88846807251356, "avg_mpg_min": 30.277303737481972, "avg_mpg_max": 31.651313299191177}, "sampler_perf": {"mean_env_wait_ms": 1574.7721148055796, "mean_processing_ms": 429.84474491087764, "mean_inference_ms": 1.8584101578272065}, "off_policy_estimator": {}, "info": {"num_steps_trained": 182005504, "num_steps_sampled": 2550000, "sample_time_ms": 2082043.741, "load_time_ms": 464.498, "grad_time_ms": 190333.963, "update_time_ms": 7.99, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5276.20654296875, "policy_loss": 0.0004012341669294983, "vf_loss": 5276.2060546875, "vf_explained_var": 0.10122519731521606, "vf_preds": 134.0868682861328, "kl": 0.0057261232286691666, "entropy": -0.08325333148241043, "entropy_coeff": 0.0, "advantages": 7.487650691473391e-06, "rewards": 1.8882923126220703}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2550000, "episodes_total": 2550, "training_iteration": 150, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_21-47-58", "timestamp": 1599342478, "time_this_iter_s": 2262.3824899196625, "time_total_s": 333083.52710032463, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc6721e0>", "on_episode_step": ".on_episode_step at 0x7f3b003b3e18>", "on_episode_end": ".on_episode_end at 0x7f3b003b3ae8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3b003b3ea0>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 333083.52710032463, "timesteps_since_restore": 2550000, "iterations_since_restore": 150, "perf": {"cpu_util_percent": 89.45733788395904, "ram_util_percent": 30.62143965249767}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 143272.81171143663, "episode_reward_min": 139936.87533468593, "episode_reward_mean": 141817.83174222746, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 626.5424348176974}, "policy_reward_mean": {"av": 449.34517836008825}, "custom_metrics": {"avg_speed_mean": 4.94024544042265, "avg_speed_min": 4.85674125094646, "avg_speed_max": 5.015145692424418, "avg_speed_avs_mean": 4.931254661685482, "avg_speed_avs_min": 4.842026601171827, "avg_speed_avs_max": 5.006035977378459, "avg_headway_avs_mean": 9.157934048213583, "avg_headway_avs_min": 8.607204395313987, "avg_headway_avs_max": 9.810730634370188, "std_headway_avs_mean": 7.437329386582215, "std_headway_avs_min": 6.987246280232479, "std_headway_avs_max": 7.914646173168708, "avg_accel_avs_mean": 0.28077549768722837, "avg_accel_avs_min": 0.27252718016460176, "avg_accel_avs_max": 0.2909945398993538, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.096738437690224, "avg_mpg_per_veh_min": 30.420018137666244, "avg_mpg_per_veh_max": 31.818847141882628, "num_cars_mean": 941.2166000000001, "num_cars_min": 932.219, "num_cars_max": 950.337, "avg_mpg_mean": 30.942170999916424, "avg_mpg_min": 30.277303737481972, "avg_mpg_max": 31.651313299191177}, "sampler_perf": {"mean_env_wait_ms": 1575.3614711663322, "mean_processing_ms": 429.5918222211645, "mean_inference_ms": 1.8588107850202633}, "off_policy_estimator": {}, "info": {"num_steps_trained": 183247104, "num_steps_sampled": 2567000, "sample_time_ms": 2083886.116, "load_time_ms": 464.827, "grad_time_ms": 190551.905, "update_time_ms": 8.301, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5291.154296875, "policy_loss": 0.0004904247471131384, "vf_loss": 5291.15380859375, "vf_explained_var": 0.10748223215341568, "vf_preds": 134.0865936279297, "kl": 0.00586106488481164, "entropy": -0.09390217065811157, "entropy_coeff": 0.0, "advantages": -9.295727977587376e-06, "rewards": 1.8804163932800293}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2567000, "episodes_total": 2567, "training_iteration": 151, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_22-25-39", "timestamp": 1599344739, "time_this_iter_s": 2261.1584615707397, "time_total_s": 335344.68556189537, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc71eb70>", "on_episode_step": ".on_episode_step at 0x7f3b734211e0>", "on_episode_end": ".on_episode_end at 0x7f3b73421268>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3b73421378>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 335344.68556189537, "timesteps_since_restore": 2567000, "iterations_since_restore": 151, "perf": {"cpu_util_percent": 89.50928571428572, "ram_util_percent": 30.5727950310559}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 143272.81171143663, "episode_reward_min": 140499.66155621628, "episode_reward_mean": 142006.78717021534, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 626.5424348176974}, "policy_reward_mean": {"av": 449.715891852346}, "custom_metrics": {"avg_speed_mean": 4.947665323706056, "avg_speed_min": 4.887109743557214, "avg_speed_max": 5.015145692424418, "avg_speed_avs_mean": 4.938975665322619, "avg_speed_avs_min": 4.875762959657515, "avg_speed_avs_max": 5.006035977378459, "avg_headway_avs_mean": 9.144417414706913, "avg_headway_avs_min": 8.607204395313987, "avg_headway_avs_max": 9.810730634370188, "std_headway_avs_mean": 7.453442795436215, "std_headway_avs_min": 6.987246280232479, "std_headway_avs_max": 7.949765870255804, "avg_accel_avs_mean": 0.2794613781296164, "avg_accel_avs_min": 0.27114435218677846, "avg_accel_avs_max": 0.2909945398993538, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.15242100599542, "avg_mpg_per_veh_min": 30.668861750879543, "avg_mpg_per_veh_max": 31.818847141882628, "num_cars_mean": 940.7443299999999, "num_cars_min": 932.219, "num_cars_max": 946.873, "avg_mpg_mean": 30.996484600176732, "avg_mpg_min": 30.525897977895994, "avg_mpg_max": 31.651313299191177}, "sampler_perf": {"mean_env_wait_ms": 1575.8041746639553, "mean_processing_ms": 429.41548461539526, "mean_inference_ms": 1.8590523511105774}, "off_policy_estimator": {}, "info": {"num_steps_trained": 184489600, "num_steps_sampled": 2584000, "sample_time_ms": 2083648.28, "load_time_ms": 468.751, "grad_time_ms": 190588.793, "update_time_ms": 8.526, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5243.86767578125, "policy_loss": 0.0005267212982289493, "vf_loss": 5243.86669921875, "vf_explained_var": 0.11362733691930771, "vf_preds": 133.10397338867188, "kl": 0.00561106251552701, "entropy": -0.13421764969825745, "entropy_coeff": 0.0, "advantages": -7.227213700389257e-06, "rewards": 1.8739386796951294}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2584000, "episodes_total": 2584, "training_iteration": 152, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_23-03-03", "timestamp": 1599346983, "time_this_iter_s": 2243.519464969635, "time_total_s": 337588.205026865, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5e9ae8>", "on_episode_step": ".on_episode_step at 0x7f3afc5e9730>", "on_episode_end": ".on_episode_end at 0x7f3afc5e99d8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5e9e18>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 337588.205026865, "timesteps_since_restore": 2584000, "iterations_since_restore": 152, "perf": {"cpu_util_percent": 88.0526282853567, "ram_util_percent": 30.595494367959954}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 143679.82212721324, "episode_reward_min": 141071.81699997315, "episode_reward_mean": 142327.19247852764, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 626.5424348176974}, "policy_reward_mean": {"av": 450.10338850298103}, "custom_metrics": {"avg_speed_mean": 4.963452912332622, "avg_speed_min": 4.888691928584528, "avg_speed_max": 5.039316990236365, "avg_speed_avs_mean": 4.95591558044033, "avg_speed_avs_min": 4.879373546451901, "avg_speed_avs_max": 5.035182256495821, "avg_headway_avs_mean": 9.147070165901926, "avg_headway_avs_min": 8.607204395313987, "avg_headway_avs_max": 9.810730634370188, "std_headway_avs_mean": 7.4794969147264245, "std_headway_avs_min": 6.987246280232479, "std_headway_avs_max": 7.949765870255804, "avg_accel_avs_mean": 0.27838942201727407, "avg_accel_avs_min": 0.2667491988956737, "avg_accel_avs_max": 0.2909945398993538, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.24315879453453, "avg_mpg_per_veh_min": 30.668861750879543, "avg_mpg_per_veh_max": 31.818847141882628, "num_cars_mean": 939.5470499999999, "num_cars_min": 932.219, "num_cars_max": 946.873, "avg_mpg_mean": 31.08425820970376, "avg_mpg_min": 30.525897977895994, "avg_mpg_max": 31.651313299191177}, "sampler_perf": {"mean_env_wait_ms": 1576.0865213752022, "mean_processing_ms": 429.3503661121722, "mean_inference_ms": 1.8591236404410247}, "off_policy_estimator": {}, "info": {"num_steps_trained": 185724160, "num_steps_sampled": 2601000, "sample_time_ms": 2080214.389, "load_time_ms": 465.021, "grad_time_ms": 190771.6, "update_time_ms": 8.533, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5289.3095703125, "policy_loss": 0.0004386658256407827, "vf_loss": 5289.30908203125, "vf_explained_var": 0.12712512910366058, "vf_preds": 133.10397338867188, "kl": 0.006179611198604107, "entropy": -0.1597992181777954, "entropy_coeff": 0.0, "advantages": 8.751716791266517e-07, "rewards": 1.9039766788482666}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2601000, "episodes_total": 2601, "training_iteration": 153, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-05_23-40-08", "timestamp": 1599349208, "time_this_iter_s": 2224.7826113700867, "time_total_s": 339812.9876382351, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5e90d0>", "on_episode_step": ".on_episode_step at 0x7f3afc5e96a8>", "on_episode_end": ".on_episode_end at 0x7f3afc5e9950>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc71eb70>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 339812.9876382351, "timesteps_since_restore": 2601000, "iterations_since_restore": 153, "perf": {"cpu_util_percent": 88.12754812243608, "ram_util_percent": 30.626254338908176}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144601.38914501257, "episode_reward_min": 141103.30398590709, "episode_reward_mean": 142700.45973092326, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 626.5424348176974}, "policy_reward_mean": {"av": 450.5856006660032}, "custom_metrics": {"avg_speed_mean": 4.97567105439265, "avg_speed_min": 4.895899699879509, "avg_speed_max": 5.0714895371433375, "avg_speed_avs_mean": 4.969092649319922, "avg_speed_avs_min": 4.884789130248616, "avg_speed_avs_max": 5.0673862819878535, "avg_headway_avs_mean": 9.082045435114116, "avg_headway_avs_min": 8.504501963013157, "avg_headway_avs_max": 9.810730634370188, "std_headway_avs_mean": 7.447571438615448, "std_headway_avs_min": 6.987246280232479, "std_headway_avs_max": 7.949765870255804, "avg_accel_avs_mean": 0.27645621002524146, "avg_accel_avs_min": 0.2623381056913155, "avg_accel_avs_max": 0.28878405513677824, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.337179113969437, "avg_mpg_per_veh_min": 30.668861750879543, "avg_mpg_per_veh_max": 32.02981153651088, "num_cars_mean": 938.99603, "num_cars_min": 932.219, "num_cars_max": 946.873, "avg_mpg_mean": 31.176778256187617, "avg_mpg_min": 30.525897977895994, "avg_mpg_max": 31.88435470035306}, "sampler_perf": {"mean_env_wait_ms": 1576.223733086538, "mean_processing_ms": 429.38998180098235, "mean_inference_ms": 1.8590427177026543}, "off_policy_estimator": {}, "info": {"num_steps_trained": 186961664, "num_steps_sampled": 2618000, "sample_time_ms": 2073389.529, "load_time_ms": 462.661, "grad_time_ms": 187848.108, "update_time_ms": 8.395, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5372.9228515625, "policy_loss": 0.00036709484993480146, "vf_loss": 5372.9228515625, "vf_explained_var": 0.13931848108768463, "vf_preds": 133.2057342529297, "kl": 0.0056692627258598804, "entropy": -0.16658541560173035, "entropy_coeff": 0.0, "advantages": -5.257945758785354e-07, "rewards": 1.9082921743392944}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2618000, "episodes_total": 2618, "training_iteration": 154, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_00-17-03", "timestamp": 1599351423, "time_this_iter_s": 2214.6273913383484, "time_total_s": 342027.61502957344, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afff9bc80>", "on_episode_step": ".on_episode_step at 0x7f3afff9bbf8>", "on_episode_end": ".on_episode_end at 0x7f3afff9bb70>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afff9b8c8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 342027.61502957344, "timesteps_since_restore": 2618000, "iterations_since_restore": 154, "perf": {"cpu_util_percent": 87.22551505546751, "ram_util_percent": 30.652234548335972}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 145226.41214664583, "episode_reward_min": 141103.30398590709, "episode_reward_mean": 143095.7736875375, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 626.5424348176974}, "policy_reward_mean": {"av": 451.1216068333462}, "custom_metrics": {"avg_speed_mean": 4.988946264529824, "avg_speed_min": 4.895899699879509, "avg_speed_max": 5.0714895371433375, "avg_speed_avs_mean": 4.983230731841909, "avg_speed_avs_min": 4.884789130248616, "avg_speed_avs_max": 5.068705289519993, "avg_headway_avs_mean": 9.00022115854302, "avg_headway_avs_min": 8.138508590329868, "avg_headway_avs_max": 9.810730634370188, "std_headway_avs_mean": 7.418773931572794, "std_headway_avs_min": 6.881146955150767, "std_headway_avs_max": 7.949765870255804, "avg_accel_avs_mean": 0.27298817629137706, "avg_accel_avs_min": 0.2609996032509862, "avg_accel_avs_max": 0.28486126652985944, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.435978860875704, "avg_mpg_per_veh_min": 30.668861750879543, "avg_mpg_per_veh_max": 32.02981153651088, "num_cars_mean": 938.5748500000002, "num_cars_min": 932.219, "num_cars_max": 946.873, "avg_mpg_mean": 31.27197368333912, "avg_mpg_min": 30.525897977895994, "avg_mpg_max": 31.88435470035306}, "sampler_perf": {"mean_env_wait_ms": 1576.274356948538, "mean_processing_ms": 429.50514129832976, "mean_inference_ms": 1.858863591976024}, "off_policy_estimator": {}, "info": {"num_steps_trained": 188201600, "num_steps_sampled": 2635000, "sample_time_ms": 2065620.791, "load_time_ms": 464.089, "grad_time_ms": 184966.215, "update_time_ms": 8.81, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5276.9755859375, "policy_loss": 0.0003000999568030238, "vf_loss": 5276.9755859375, "vf_explained_var": 0.1571098119020462, "vf_preds": 131.16030883789062, "kl": 0.005574868526309729, "entropy": -0.15452176332473755, "entropy_coeff": 0.0, "advantages": 1.2256879927008413e-05, "rewards": 1.9157698154449463}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2635000, "episodes_total": 2635, "training_iteration": 155, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_00-54-03", "timestamp": 1599353643, "time_this_iter_s": 2220.381629228592, "time_total_s": 344247.99665880203, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5e9510>", "on_episode_step": ".on_episode_step at 0x7f3afc5e9ea0>", "on_episode_end": ".on_episode_end at 0x7f3afc5e9ae8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5e9950>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 344247.99665880203, "timesteps_since_restore": 2635000, "iterations_since_restore": 155, "perf": {"cpu_util_percent": 87.68665401644529, "ram_util_percent": 30.660025300442758}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 145226.41214664583, "episode_reward_min": 141103.30398590709, "episode_reward_mean": 143385.81579602775, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 626.5424348176974}, "policy_reward_mean": {"av": 451.50932328629176}, "custom_metrics": {"avg_speed_mean": 4.999836042057645, "avg_speed_min": 4.895899699879509, "avg_speed_max": 5.07453204624435, "avg_speed_avs_mean": 4.994983965942809, "avg_speed_avs_min": 4.884789130248616, "avg_speed_avs_max": 5.076756025270293, "avg_headway_avs_mean": 8.972392283616458, "avg_headway_avs_min": 8.138508590329868, "avg_headway_avs_max": 9.810730634370188, "std_headway_avs_mean": 7.44273335372793, "std_headway_avs_min": 6.881146955150767, "std_headway_avs_max": 7.949765870255804, "avg_accel_avs_mean": 0.2708334234020209, "avg_accel_avs_min": 0.2609996032509862, "avg_accel_avs_max": 0.2843255930949957, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.52830945631304, "avg_mpg_per_veh_min": 30.668861750879543, "avg_mpg_per_veh_max": 32.158580396008865, "num_cars_mean": 937.94343, "num_cars_min": 931.605, "num_cars_max": 946.873, "avg_mpg_mean": 31.362383420178684, "avg_mpg_min": 30.525897977895994, "avg_mpg_max": 31.964145891810812}, "sampler_perf": {"mean_env_wait_ms": 1576.281650612321, "mean_processing_ms": 429.64492180565816, "mean_inference_ms": 1.8586337966449902}, "off_policy_estimator": {}, "info": {"num_steps_trained": 189437824, "num_steps_sampled": 2652000, "sample_time_ms": 2059942.5, "load_time_ms": 464.117, "grad_time_ms": 182745.726, "update_time_ms": 8.72, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5350.1630859375, "policy_loss": 0.00046334072249010205, "vf_loss": 5350.16259765625, "vf_explained_var": 0.16611163318157196, "vf_preds": 128.68797302246094, "kl": 0.005416943691670895, "entropy": -0.15154029428958893, "entropy_coeff": 0.0, "advantages": 1.0370038580731489e-05, "rewards": 1.920424222946167}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2652000, "episodes_total": 2652, "training_iteration": 156, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_01-31-29", "timestamp": 1599355889, "time_this_iter_s": 2246.0016112327576, "time_total_s": 346493.9982700348, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5e9c80>", "on_episode_step": ".on_episode_step at 0x7f3afc5e9b70>", "on_episode_end": ".on_episode_end at 0x7f3afc5e9400>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5e99d8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 346493.9982700348, "timesteps_since_restore": 2652000, "iterations_since_restore": 156, "perf": {"cpu_util_percent": 89.00803125, "ram_util_percent": 30.683468750000003}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 145241.63119426198, "episode_reward_min": 141140.62089124965, "episode_reward_mean": 143799.0443324499, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 626.2231009119234}, "policy_reward_mean": {"av": 452.1698142646684}, "custom_metrics": {"avg_speed_mean": 5.0161002614795285, "avg_speed_min": 4.895899699879509, "avg_speed_max": 5.086629064376337, "avg_speed_avs_mean": 5.0125663384503, "avg_speed_avs_min": 4.884789130248616, "avg_speed_avs_max": 5.084185293433197, "avg_headway_avs_mean": 8.952900279203217, "avg_headway_avs_min": 8.138508590329868, "avg_headway_avs_max": 9.581728018919847, "std_headway_avs_mean": 7.477737588556687, "std_headway_avs_min": 6.881146955150767, "std_headway_avs_max": 7.949765870255804, "avg_accel_avs_mean": 0.26808857527572066, "avg_accel_avs_min": 0.26071336472552636, "avg_accel_avs_max": 0.2803967333708367, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.660429676794678, "avg_mpg_per_veh_min": 30.89918671424601, "avg_mpg_per_veh_max": 32.28517574411381, "num_cars_mean": 936.97671, "num_cars_min": 929.243, "num_cars_max": 946.873, "avg_mpg_mean": 31.490888062194795, "avg_mpg_min": 30.71273710556874, "avg_mpg_max": 32.10587500944485}, "sampler_perf": {"mean_env_wait_ms": 1576.2754234729268, "mean_processing_ms": 429.7811977696276, "mean_inference_ms": 1.8583981042613809}, "off_policy_estimator": {}, "info": {"num_steps_trained": 190670080, "num_steps_sampled": 2669000, "sample_time_ms": 2058792.043, "load_time_ms": 465.312, "grad_time_ms": 182269.68, "update_time_ms": 8.644, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5656.56591796875, "policy_loss": 0.00043920797179453075, "vf_loss": 5656.56494140625, "vf_explained_var": 0.1833042949438095, "vf_preds": 127.45323181152344, "kl": 0.005608178675174713, "entropy": -0.17201215028762817, "entropy_coeff": 0.0, "advantages": 6.506465979327913e-06, "rewards": 1.92710542678833}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2669000, "episodes_total": 2669, "training_iteration": 157, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_02-09-03", "timestamp": 1599358143, "time_this_iter_s": 2253.5564727783203, "time_total_s": 348747.5547428131, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc597620>", "on_episode_step": ".on_episode_step at 0x7f3afc5976a8>", "on_episode_end": ".on_episode_end at 0x7f3afc597b70>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc597bf8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 348747.5547428131, "timesteps_since_restore": 2669000, "iterations_since_restore": 157, "perf": {"cpu_util_percent": 88.84996884735203, "ram_util_percent": 30.560778816199377}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 145241.63119426198, "episode_reward_min": 142407.05449700708, "episode_reward_mean": 144007.5705370614, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 626.2231009119234}, "policy_reward_mean": {"av": 452.48403989524707}, "custom_metrics": {"avg_speed_mean": 5.028109511668822, "avg_speed_min": 4.974063955893394, "avg_speed_max": 5.086629064376337, "avg_speed_avs_mean": 5.025339157707138, "avg_speed_avs_min": 4.964759541598978, "avg_speed_avs_max": 5.084185293433197, "avg_headway_avs_mean": 8.998319261605007, "avg_headway_avs_min": 8.138508590329868, "avg_headway_avs_max": 10.089024557649012, "std_headway_avs_mean": 7.525730335240293, "std_headway_avs_min": 6.881146955150767, "std_headway_avs_max": 8.184721185051366, "avg_accel_avs_mean": 0.26652126490462213, "avg_accel_avs_min": 0.26071336472552636, "avg_accel_avs_max": 0.27646721461150414, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.72515028099719, "avg_mpg_per_veh_min": 31.057743695152325, "avg_mpg_per_veh_max": 32.28517574411381, "num_cars_mean": 936.0129999999999, "num_cars_min": 927.126, "num_cars_max": 943.645, "avg_mpg_mean": 31.555466979492746, "avg_mpg_min": 30.889764303342137, "avg_mpg_max": 32.10587500944485}, "sampler_perf": {"mean_env_wait_ms": 1576.2688363894256, "mean_processing_ms": 429.9157000228738, "mean_inference_ms": 1.8581538221584324}, "off_policy_estimator": {}, "info": {"num_steps_trained": 191902336, "num_steps_sampled": 2686000, "sample_time_ms": 2057238.198, "load_time_ms": 463.002, "grad_time_ms": 182240.346, "update_time_ms": 8.66, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5944.27099609375, "policy_loss": 0.0005457103252410889, "vf_loss": 5944.2705078125, "vf_explained_var": 0.1565675437450409, "vf_preds": 129.26223754882812, "kl": 0.007472957484424114, "entropy": -0.17486968636512756, "entropy_coeff": 0.0, "advantages": -2.987374045915203e-06, "rewards": 1.9082528352737427}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2686000, "episodes_total": 2686, "training_iteration": 158, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_02-46-22", "timestamp": 1599360382, "time_this_iter_s": 2238.7889709472656, "time_total_s": 350986.3437137604, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5e98c8>", "on_episode_step": ".on_episode_step at 0x7f3afc5e9488>", "on_episode_end": ".on_episode_end at 0x7f3afc5e9d08>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5e9730>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 350986.3437137604, "timesteps_since_restore": 2686000, "iterations_since_restore": 158, "perf": {"cpu_util_percent": 88.90667920978363, "ram_util_percent": 30.715961116337407}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 145241.63119426198, "episode_reward_min": 142407.05449700708, "episode_reward_mean": 144194.47379706518, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 626.2231009119234}, "policy_reward_mean": {"av": 452.8578681481897}, "custom_metrics": {"avg_speed_mean": 5.031003238339303, "avg_speed_min": 4.974063955893394, "avg_speed_max": 5.086629064376337, "avg_speed_avs_mean": 5.028642231720649, "avg_speed_avs_min": 4.964759541598978, "avg_speed_avs_max": 5.084185293433197, "avg_headway_avs_mean": 8.938676479697248, "avg_headway_avs_min": 8.138508590329868, "avg_headway_avs_max": 10.089024557649012, "std_headway_avs_mean": 7.4856157192811805, "std_headway_avs_min": 6.881146955150767, "std_headway_avs_max": 8.184721185051366, "avg_accel_avs_mean": 0.2648223619321957, "avg_accel_avs_min": 0.2566304744785774, "avg_accel_avs_max": 0.27420204220978556, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.785410026523994, "avg_mpg_per_veh_min": 31.198488214619307, "avg_mpg_per_veh_max": 32.28517574411381, "num_cars_mean": 936.2525499999999, "num_cars_min": 927.126, "num_cars_max": 944.006, "avg_mpg_mean": 31.616611396673452, "avg_mpg_min": 31.03397752114315, "avg_mpg_max": 32.10587500944485}, "sampler_perf": {"mean_env_wait_ms": 1576.2840720019394, "mean_processing_ms": 430.0487028812354, "mean_inference_ms": 1.85793328795939}, "off_policy_estimator": {}, "info": {"num_steps_trained": 193139072, "num_steps_sampled": 2703000, "sample_time_ms": 2058114.513, "load_time_ms": 463.631, "grad_time_ms": 182193.914, "update_time_ms": 8.652, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5937.85302734375, "policy_loss": 0.0004890543059445918, "vf_loss": 5937.8525390625, "vf_explained_var": 0.1622769832611084, "vf_preds": 128.49966430664062, "kl": 0.005808417219668627, "entropy": -0.1731416881084442, "entropy_coeff": 0.0, "advantages": 2.9798682135151466e-06, "rewards": 1.9150707721710205}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2703000, "episodes_total": 2703, "training_iteration": 159, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_03-23-48", "timestamp": 1599362628, "time_this_iter_s": 2246.6886315345764, "time_total_s": 353233.03234529495, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc541d08>", "on_episode_step": ".on_episode_step at 0x7f3afc541f28>", "on_episode_end": ".on_episode_end at 0x7f3afc541400>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc541b70>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 353233.03234529495, "timesteps_since_restore": 2703000, "iterations_since_restore": 159, "perf": {"cpu_util_percent": 88.84184375000001, "ram_util_percent": 30.693656249999993}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 145241.63119426198, "episode_reward_min": 142328.1258256724, "episode_reward_mean": 143979.88727459605, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 624.7978638164899}, "policy_reward_mean": {"av": 452.69576253606664}, "custom_metrics": {"avg_speed_mean": 5.024388342887877, "avg_speed_min": 4.954376704632854, "avg_speed_max": 5.086629064376337, "avg_speed_avs_mean": 5.021553967953454, "avg_speed_avs_min": 4.94371168355734, "avg_speed_avs_max": 5.084185293433197, "avg_headway_avs_mean": 9.00501650038541, "avg_headway_avs_min": 8.138508590329868, "avg_headway_avs_max": 10.089024557649012, "std_headway_avs_mean": 7.526265014463692, "std_headway_avs_min": 6.881146955150767, "std_headway_avs_max": 8.184721185051366, "avg_accel_avs_mean": 0.2642529337428999, "avg_accel_avs_min": 0.2566304744785774, "avg_accel_avs_max": 0.26889328632527787, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.75004287563159, "avg_mpg_per_veh_min": 31.1061457330397, "avg_mpg_per_veh_max": 32.28517574411381, "num_cars_mean": 936.3759800000001, "num_cars_min": 927.126, "num_cars_max": 944.006, "avg_mpg_mean": 31.58369279008694, "avg_mpg_min": 30.96819486560197, "avg_mpg_max": 32.10587500944485}, "sampler_perf": {"mean_env_wait_ms": 1576.3483246074982, "mean_processing_ms": 430.18144593540904, "mean_inference_ms": 1.8577700413058094}, "off_policy_estimator": {}, "info": {"num_steps_trained": 194376960, "num_steps_sampled": 2720000, "sample_time_ms": 2057393.698, "load_time_ms": 467.353, "grad_time_ms": 182128.0, "update_time_ms": 8.712, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5986.671875, "policy_loss": 0.0005414953920990229, "vf_loss": 5986.671875, "vf_explained_var": 0.14080049097537994, "vf_preds": 128.38491821289062, "kl": 0.005969292484223843, "entropy": -0.18452967703342438, "entropy_coeff": 0.0, "advantages": -1.6981869066512445e-07, "rewards": 1.8939182758331299}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2720000, "episodes_total": 2720, "training_iteration": 160, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_04-01-23", "timestamp": 1599364883, "time_this_iter_s": 2254.5748403072357, "time_total_s": 355487.6071856022, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc3b0158>", "on_episode_step": ".on_episode_step at 0x7f3afc3b00d0>", "on_episode_end": ".on_episode_end at 0x7f3afc3b0a60>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc3b0048>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 355487.6071856022, "timesteps_since_restore": 2720000, "iterations_since_restore": 160, "perf": {"cpu_util_percent": 89.61024602927436, "ram_util_percent": 30.731983805668015}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 145241.63119426198, "episode_reward_min": 141742.08719471694, "episode_reward_mean": 143610.22296926248, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 624.7978638164899}, "policy_reward_mean": {"av": 452.2871723647722}, "custom_metrics": {"avg_speed_mean": 5.013679618139721, "avg_speed_min": 4.922921398156619, "avg_speed_max": 5.086629064376337, "avg_speed_avs_mean": 5.009628978586014, "avg_speed_avs_min": 4.91455597115897, "avg_speed_avs_max": 5.084185293433197, "avg_headway_avs_mean": 9.125096724981, "avg_headway_avs_min": 8.339201060353322, "avg_headway_avs_max": 10.089024557649012, "std_headway_avs_mean": 7.617749712760255, "std_headway_avs_min": 7.086393039598907, "std_headway_avs_max": 8.184721185051366, "avg_accel_avs_mean": 0.26395650183148145, "avg_accel_avs_min": 0.2566304744785774, "avg_accel_avs_max": 0.26889328632527787, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.680383517176498, "avg_mpg_per_veh_min": 30.977163161597925, "avg_mpg_per_veh_max": 32.28517574411381, "num_cars_mean": 936.3943599999999, "num_cars_min": 927.126, "num_cars_max": 944.006, "avg_mpg_mean": 31.51518926421805, "avg_mpg_min": 30.786947179595497, "avg_mpg_max": 32.10587500944485}, "sampler_perf": {"mean_env_wait_ms": 1576.4287254984247, "mean_processing_ms": 430.3118556137941, "mean_inference_ms": 1.8576284950734459}, "off_policy_estimator": {}, "info": {"num_steps_trained": 195616640, "num_steps_sampled": 2737000, "sample_time_ms": 2055338.553, "load_time_ms": 467.98, "grad_time_ms": 182024.964, "update_time_ms": 8.546, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5890.388671875, "policy_loss": 0.0005432158359326422, "vf_loss": 5890.388671875, "vf_explained_var": 0.13536721467971802, "vf_preds": 127.30782318115234, "kl": 0.006209888495504856, "entropy": -0.21213002502918243, "entropy_coeff": 0.0, "advantages": -3.2719842693040846e-06, "rewards": 1.8871225118637085}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2737000, "episodes_total": 2737, "training_iteration": 161, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_04-38-43", "timestamp": 1599367123, "time_this_iter_s": 2239.594868183136, "time_total_s": 357727.2020537853, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5f9488>", "on_episode_step": ".on_episode_step at 0x7f3afc5f9ea0>", "on_episode_end": ".on_episode_end at 0x7f3afc5f9840>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5f97b8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 357727.2020537853, "timesteps_since_restore": 2737000, "iterations_since_restore": 161, "perf": {"cpu_util_percent": 88.69156739811912, "ram_util_percent": 30.70476489028213}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 145241.63119426198, "episode_reward_min": 141371.29391563436, "episode_reward_mean": 143164.53118004752, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 629.5062945919456}, "policy_reward_mean": {"av": 451.8226698859038}, "custom_metrics": {"avg_speed_mean": 4.998283597969879, "avg_speed_min": 4.908720435811445, "avg_speed_max": 5.086629064376337, "avg_speed_avs_mean": 4.992780276068098, "avg_speed_avs_min": 4.903296001438167, "avg_speed_avs_max": 5.084185293433197, "avg_headway_avs_mean": 9.209486779550904, "avg_headway_avs_min": 8.339201060353322, "avg_headway_avs_max": 10.089024557649012, "std_headway_avs_mean": 7.679441765834912, "std_headway_avs_min": 7.086393039598907, "std_headway_avs_max": 8.251493388074712, "avg_accel_avs_mean": 0.2631087831373337, "avg_accel_avs_min": 0.2566304744785774, "avg_accel_avs_max": 0.26889328632527787, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.592561520111268, "avg_mpg_per_veh_min": 30.977163161597925, "avg_mpg_per_veh_max": 32.28517574411381, "num_cars_mean": 936.9093, "num_cars_min": 927.126, "num_cars_max": 946.592, "avg_mpg_mean": 31.429749945326936, "avg_mpg_min": 30.786947179595497, "avg_mpg_max": 32.10587500944485}, "sampler_perf": {"mean_env_wait_ms": 1576.5027445418987, "mean_processing_ms": 430.4384772491202, "mean_inference_ms": 1.8574858149054287}, "off_policy_estimator": {}, "info": {"num_steps_trained": 196856832, "num_steps_sampled": 2754000, "sample_time_ms": 2054341.988, "load_time_ms": 461.29, "grad_time_ms": 182615.726, "update_time_ms": 8.249, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5771.91796875, "policy_loss": 0.0005915772635489702, "vf_loss": 5771.916015625, "vf_explained_var": 0.13509027659893036, "vf_preds": 125.51457977294922, "kl": 0.006330294068902731, "entropy": -0.22208034992218018, "entropy_coeff": 0.0, "advantages": -2.6372456432000035e-06, "rewards": 1.879483938217163}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2754000, "episodes_total": 2754, "training_iteration": 162, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_05-16-02", "timestamp": 1599369362, "time_this_iter_s": 2239.306710243225, "time_total_s": 359966.50876402855, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc0a9378>", "on_episode_step": ".on_episode_step at 0x7f3afc0a9400>", "on_episode_end": ".on_episode_end at 0x7f3afc0a9488>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc0a9510>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 359966.50876402855, "timesteps_since_restore": 2754000, "iterations_since_restore": 162, "perf": {"cpu_util_percent": 88.44404388714733, "ram_util_percent": 30.74398119122257}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144807.0884007311, "episode_reward_min": 141371.29391563436, "episode_reward_mean": 142801.98797772883, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 629.5062945919456}, "policy_reward_mean": {"av": 451.31945253856946}, "custom_metrics": {"avg_speed_mean": 4.983748179333287, "avg_speed_min": 4.908720435811445, "avg_speed_max": 5.080923347293139, "avg_speed_avs_mean": 4.977135161004281, "avg_speed_avs_min": 4.903296001438167, "avg_speed_avs_max": 5.076686223774098, "avg_headway_avs_mean": 9.251652284523736, "avg_headway_avs_min": 8.339201060353322, "avg_headway_avs_max": 10.089024557649012, "std_headway_avs_mean": 7.700719406935792, "std_headway_avs_min": 7.086393039598907, "std_headway_avs_max": 8.251493388074712, "avg_accel_avs_mean": 0.26335549253370055, "avg_accel_avs_min": 0.2566304744785774, "avg_accel_avs_max": 0.270043701179675, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.505666335988035, "avg_mpg_per_veh_min": 30.977163161597925, "avg_mpg_per_veh_max": 32.10842190273064, "num_cars_mean": 937.5560700000001, "num_cars_min": 927.126, "num_cars_max": 946.592, "avg_mpg_mean": 31.344178099165518, "avg_mpg_min": 30.786947179595497, "avg_mpg_max": 31.95523921654681}, "sampler_perf": {"mean_env_wait_ms": 1576.5785000975334, "mean_processing_ms": 430.56155656478006, "mean_inference_ms": 1.857346977337825}, "off_policy_estimator": {}, "info": {"num_steps_trained": 198095488, "num_steps_sampled": 2771000, "sample_time_ms": 2056398.331, "load_time_ms": 464.418, "grad_time_ms": 182162.989, "update_time_ms": 8.261, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5759.408203125, "policy_loss": 0.000607288908213377, "vf_loss": 5759.4072265625, "vf_explained_var": 0.1360379159450531, "vf_preds": 126.58551025390625, "kl": 0.005594143643975258, "entropy": -0.1943962574005127, "entropy_coeff": 0.0, "advantages": -2.139299795089755e-06, "rewards": 1.8867393732070923}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2771000, "episodes_total": 2771, "training_iteration": 163, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_05-53-23", "timestamp": 1599371603, "time_this_iter_s": 2240.8340272903442, "time_total_s": 362207.3427913189, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc597b70>", "on_episode_step": ".on_episode_step at 0x7f3afc704950>", "on_episode_end": ".on_episode_end at 0x7f3afc6c02f0>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc6c0730>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 362207.3427913189, "timesteps_since_restore": 2771000, "iterations_since_restore": 163, "perf": {"cpu_util_percent": 88.59150736446256, "ram_util_percent": 30.782544656847385}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144807.0884007311, "episode_reward_min": 141371.29391563436, "episode_reward_mean": 142669.50960912302, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 629.5062945919456}, "policy_reward_mean": {"av": 451.20022014270387}, "custom_metrics": {"avg_speed_mean": 4.976751434849357, "avg_speed_min": 4.908720435811445, "avg_speed_max": 5.064303166800559, "avg_speed_avs_mean": 4.969657219202402, "avg_speed_avs_min": 4.903296001438167, "avg_speed_avs_max": 5.059015821064813, "avg_headway_avs_mean": 9.257968467892297, "avg_headway_avs_min": 8.339201060353322, "avg_headway_avs_max": 10.065881236172066, "std_headway_avs_mean": 7.7171945071368295, "std_headway_avs_min": 7.086393039598907, "std_headway_avs_max": 8.251493388074712, "avg_accel_avs_mean": 0.2617796925646799, "avg_accel_avs_min": 0.2524622905872545, "avg_accel_avs_max": 0.270043701179675, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.496839033343214, "avg_mpg_per_veh_min": 30.977163161597925, "avg_mpg_per_veh_max": 32.10842190273064, "num_cars_mean": 938.0100499999999, "num_cars_min": 930.086, "num_cars_max": 946.592, "avg_mpg_mean": 31.335562448471183, "avg_mpg_min": 30.786947179595497, "avg_mpg_max": 31.95523921654681}, "sampler_perf": {"mean_env_wait_ms": 1576.636107063012, "mean_processing_ms": 430.68092670748445, "mean_inference_ms": 1.8571777501386182}, "off_policy_estimator": {}, "info": {"num_steps_trained": 199332352, "num_steps_sampled": 2788000, "sample_time_ms": 2057182.775, "load_time_ms": 465.575, "grad_time_ms": 182201.424, "update_time_ms": 8.198, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5764.94677734375, "policy_loss": 0.0005923661519773304, "vf_loss": 5764.9462890625, "vf_explained_var": 0.14144986867904663, "vf_preds": 124.80289459228516, "kl": 0.005763975903391838, "entropy": -0.22618265450000763, "entropy_coeff": 0.0, "advantages": -2.2626140889769886e-06, "rewards": 1.891985297203064}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2788000, "episodes_total": 2788, "training_iteration": 164, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_06-30-26", "timestamp": 1599373826, "time_this_iter_s": 2222.8745827674866, "time_total_s": 364430.2173740864, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc084b70>", "on_episode_step": ".on_episode_step at 0x7f3afc084bf8>", "on_episode_end": ".on_episode_end at 0x7f3afc084c80>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc084d08>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 364430.2173740864, "timesteps_since_restore": 2788000, "iterations_since_restore": 164, "perf": {"cpu_util_percent": 87.02131354594253, "ram_util_percent": 30.698610672560783}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 143336.81444082153, "episode_reward_min": 141371.29391563436, "episode_reward_mean": 142347.59337050392, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 629.5062945919456}, "policy_reward_mean": {"av": 450.7238090383885}, "custom_metrics": {"avg_speed_mean": 4.967606889444472, "avg_speed_min": 4.908720435811445, "avg_speed_max": 5.038153844714321, "avg_speed_avs_mean": 4.959383607290488, "avg_speed_avs_min": 4.903296001438167, "avg_speed_avs_max": 5.030332643707203, "avg_headway_avs_mean": 9.36973989858736, "avg_headway_avs_min": 8.7233972349713, "avg_headway_avs_max": 10.10944722340753, "std_headway_avs_mean": 7.818804023149807, "std_headway_avs_min": 7.322834001875209, "std_headway_avs_max": 8.419174082915772, "avg_accel_avs_mean": 0.2600316926889789, "avg_accel_avs_min": 0.24813460297397275, "avg_accel_avs_max": 0.270043701179675, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.4535465046379, "avg_mpg_per_veh_min": 30.977163161597925, "avg_mpg_per_veh_max": 31.93757516932034, "num_cars_mean": 938.0653600000002, "num_cars_min": 930.086, "num_cars_max": 946.592, "avg_mpg_mean": 31.2927054507142, "avg_mpg_min": 30.786947179595497, "avg_mpg_max": 31.786034435753372}, "sampler_perf": {"mean_env_wait_ms": 1576.686623430637, "mean_processing_ms": 430.7984152862729, "mean_inference_ms": 1.8569893881818789}, "off_policy_estimator": {}, "info": {"num_steps_trained": 200568448, "num_steps_sampled": 2805000, "sample_time_ms": 2058615.381, "load_time_ms": 464.106, "grad_time_ms": 182145.248, "update_time_ms": 8.113, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5693.69677734375, "policy_loss": 0.0006124472129158676, "vf_loss": 5693.69580078125, "vf_explained_var": 0.1455131322145462, "vf_preds": 122.23162078857422, "kl": 0.006378060672432184, "entropy": -0.25719431042671204, "entropy_coeff": 0.0, "advantages": -9.927515748131555e-06, "rewards": 1.8880940675735474}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2805000, "episodes_total": 2805, "training_iteration": 165, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_07-07-40", "timestamp": 1599376060, "time_this_iter_s": 2234.080819606781, "time_total_s": 366664.29819369316, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc6c0d90>", "on_episode_step": ".on_episode_step at 0x7f3afc084158>", "on_episode_end": ".on_episode_end at 0x7f3afc0840d0>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc084048>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 366664.29819369316, "timesteps_since_restore": 2805000, "iterations_since_restore": 165, "perf": {"cpu_util_percent": 87.59921433060967, "ram_util_percent": 30.783092394720303}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 143336.81444082153, "episode_reward_min": 139803.02670968787, "episode_reward_mean": 141986.3508415537, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 629.5062945919456}, "policy_reward_mean": {"av": 450.37857908251493}, "custom_metrics": {"avg_speed_mean": 4.955113523694831, "avg_speed_min": 4.862470288921142, "avg_speed_max": 5.038153844714321, "avg_speed_avs_mean": 4.946037410483966, "avg_speed_avs_min": 4.851379255370235, "avg_speed_avs_max": 5.030332643707203, "avg_headway_avs_mean": 9.453005605840143, "avg_headway_avs_min": 8.7233972349713, "avg_headway_avs_max": 10.10944722340753, "std_headway_avs_mean": 7.920445369278869, "std_headway_avs_min": 7.464815230258408, "std_headway_avs_max": 8.419174082915772, "avg_accel_avs_mean": 0.25817572307970665, "avg_accel_avs_min": 0.2475095044504249, "avg_accel_avs_max": 0.270043701179675, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.399849460454924, "avg_mpg_per_veh_min": 30.652440113036356, "avg_mpg_per_veh_max": 31.93757516932034, "num_cars_mean": 938.4177299999997, "num_cars_min": 930.086, "num_cars_max": 946.592, "avg_mpg_mean": 31.240082957504306, "avg_mpg_min": 30.522180280354334, "avg_mpg_max": 31.786034435753372}, "sampler_perf": {"mean_env_wait_ms": 1576.7400169636019, "mean_processing_ms": 430.9101886553601, "mean_inference_ms": 1.8567920183322357}, "off_policy_estimator": {}, "info": {"num_steps_trained": 201809664, "num_steps_sampled": 2822000, "sample_time_ms": 2059695.057, "load_time_ms": 464.302, "grad_time_ms": 182315.654, "update_time_ms": 8.468, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5634.3642578125, "policy_loss": 0.0006359816179610789, "vf_loss": 5634.3642578125, "vf_explained_var": 0.13477985560894012, "vf_preds": 122.30919647216797, "kl": 0.007309614680707455, "entropy": -0.2979962229728699, "entropy_coeff": 0.0, "advantages": -3.389077619431191e-06, "rewards": 1.8609780073165894}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2822000, "episodes_total": 2822, "training_iteration": 166, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_07-45-19", "timestamp": 1599378319, "time_this_iter_s": 2258.5327513217926, "time_total_s": 368922.83094501495, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5e9e18>", "on_episode_step": ".on_episode_step at 0x7f3afc5e9510>", "on_episode_end": ".on_episode_end at 0x7f3afc5e9ea0>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5e9b70>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 368922.83094501495, "timesteps_since_restore": 2822000, "iterations_since_restore": 166, "perf": {"cpu_util_percent": 87.43394466894622, "ram_util_percent": 30.810755362138643}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 143208.6234559867, "episode_reward_min": 139803.02670968787, "episode_reward_mean": 141666.08150062314, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 629.5062945919456}, "policy_reward_mean": {"av": 450.0765074997557}, "custom_metrics": {"avg_speed_mean": 4.944811997430046, "avg_speed_min": 4.833508281528948, "avg_speed_max": 5.038153844714321, "avg_speed_avs_mean": 4.934804180014273, "avg_speed_avs_min": 4.817095492805812, "avg_speed_avs_max": 5.030332643707203, "avg_headway_avs_mean": 9.549782811165008, "avg_headway_avs_min": 8.7233972349713, "avg_headway_avs_max": 10.300613375651613, "std_headway_avs_mean": 8.027275585627898, "std_headway_avs_min": 7.464815230258408, "std_headway_avs_max": 8.709787638189193, "avg_accel_avs_mean": 0.2560894466448523, "avg_accel_avs_min": 0.24525989005132276, "avg_accel_avs_max": 0.270043701179675, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.35222239918325, "avg_mpg_per_veh_min": 30.652440113036356, "avg_mpg_per_veh_max": 31.93757516932034, "num_cars_mean": 938.6086, "num_cars_min": 930.086, "num_cars_max": 947.574, "avg_mpg_mean": 31.195214126678366, "avg_mpg_min": 30.522180280354334, "avg_mpg_max": 31.786034435753372}, "sampler_perf": {"mean_env_wait_ms": 1576.8329912234415, "mean_processing_ms": 431.012564337841, "mean_inference_ms": 1.856622978872893}, "off_policy_estimator": {}, "info": {"num_steps_trained": 203050240, "num_steps_sampled": 2839000, "sample_time_ms": 2062319.285, "load_time_ms": 465.127, "grad_time_ms": 183390.51, "update_time_ms": 8.558, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5594.23046875, "policy_loss": 0.0006436194526031613, "vf_loss": 5594.22998046875, "vf_explained_var": 0.13529634475708008, "vf_preds": 122.14274597167969, "kl": 0.006527286488562822, "entropy": -0.31175562739372253, "entropy_coeff": 0.0, "advantages": -9.09320169739658e-06, "rewards": 1.8593679666519165}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2839000, "episodes_total": 2839, "training_iteration": 167, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_08-23-29", "timestamp": 1599380609, "time_this_iter_s": 2290.581828355789, "time_total_s": 371213.41277337074, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5e96a8>", "on_episode_step": ".on_episode_step at 0x7f3afc5e9ae8>", "on_episode_end": ".on_episode_end at 0x7f3afc627268>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc627158>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 371213.41277337074, "timesteps_since_restore": 2839000, "iterations_since_restore": 167, "perf": {"cpu_util_percent": 86.81456161863886, "ram_util_percent": 30.82458614347027}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 143208.6234559867, "episode_reward_min": 139803.02670968787, "episode_reward_mean": 141482.6833456849, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 622.1326928375021}, "policy_reward_mean": {"av": 450.00853481451924}, "custom_metrics": {"avg_speed_mean": 4.939894837765014, "avg_speed_min": 4.833508281528948, "avg_speed_max": 5.038153844714321, "avg_speed_avs_mean": 4.9293483925812325, "avg_speed_avs_min": 4.817095492805812, "avg_speed_avs_max": 5.030332643707203, "avg_headway_avs_mean": 9.657631072394029, "avg_headway_avs_min": 8.7233972349713, "avg_headway_avs_max": 10.57860966044639, "std_headway_avs_mean": 8.117410576464014, "std_headway_avs_min": 7.464815230258408, "std_headway_avs_max": 8.709787638189193, "avg_accel_avs_mean": 0.2538410344535161, "avg_accel_avs_min": 0.24432545147281792, "avg_accel_avs_max": 0.270043701179675, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.348257631567854, "avg_mpg_per_veh_min": 30.652440113036356, "avg_mpg_per_veh_max": 31.93757516932034, "num_cars_mean": 938.53803, "num_cars_min": 930.086, "num_cars_max": 947.574, "avg_mpg_mean": 31.194125151075223, "avg_mpg_min": 30.522180280354334, "avg_mpg_max": 31.786034435753372}, "sampler_perf": {"mean_env_wait_ms": 1577.009318495959, "mean_processing_ms": 431.08825196748677, "mean_inference_ms": 1.8565677072286266}, "off_policy_estimator": {}, "info": {"num_steps_trained": 204287232, "num_steps_sampled": 2856000, "sample_time_ms": 2068524.434, "load_time_ms": 464.526, "grad_time_ms": 185607.813, "update_time_ms": 8.453, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5469.66796875, "policy_loss": 0.0005309631233103573, "vf_loss": 5469.66748046875, "vf_explained_var": 0.13874085247516632, "vf_preds": 121.26765441894531, "kl": 0.006569193210452795, "entropy": -0.30747929215431213, "entropy_coeff": 0.0, "advantages": 7.310434853025072e-07, "rewards": 1.8699390888214111}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2856000, "episodes_total": 2856, "training_iteration": 168, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_09-02-12", "timestamp": 1599382932, "time_this_iter_s": 2323.046100616455, "time_total_s": 373536.4588739872, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc597b70>", "on_episode_step": ".on_episode_step at 0x7f3afc084048>", "on_episode_end": ".on_episode_end at 0x7f3afc0840d0>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc084158>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 373536.4588739872, "timesteps_since_restore": 2856000, "iterations_since_restore": 168, "perf": {"cpu_util_percent": 86.87790873375643, "ram_util_percent": 30.70287095799335}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 143208.6234559867, "episode_reward_min": 139803.02670968787, "episode_reward_mean": 141299.33420130168, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 624.8818127232084}, "policy_reward_mean": {"av": 450.0122112210633}, "custom_metrics": {"avg_speed_mean": 4.936841574468149, "avg_speed_min": 4.833508281528948, "avg_speed_max": 5.038153844714321, "avg_speed_avs_mean": 4.925633812300375, "avg_speed_avs_min": 4.817095492805812, "avg_speed_avs_max": 5.030332643707203, "avg_headway_avs_mean": 9.76886928094358, "avg_headway_avs_min": 8.7233972349713, "avg_headway_avs_max": 10.76117839938169, "std_headway_avs_mean": 8.218291910841852, "std_headway_avs_min": 7.498224708780641, "std_headway_avs_max": 8.709787638189193, "avg_accel_avs_mean": 0.25129867336405665, "avg_accel_avs_min": 0.24432545147281792, "avg_accel_avs_max": 0.25992133201718765, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.346386981686013, "avg_mpg_per_veh_min": 30.652440113036356, "avg_mpg_per_veh_max": 31.93757516932034, "num_cars_mean": 938.2936400000002, "num_cars_min": 928.659, "num_cars_max": 947.574, "avg_mpg_mean": 31.191077188446098, "avg_mpg_min": 30.522180280354334, "avg_mpg_max": 31.786034435753372}, "sampler_perf": {"mean_env_wait_ms": 1577.2809317854471, "mean_processing_ms": 431.12015007257213, "mean_inference_ms": 1.8566177066937188}, "off_policy_estimator": {}, "info": {"num_steps_trained": 205521152, "num_steps_sampled": 2873000, "sample_time_ms": 2073145.623, "load_time_ms": 465.328, "grad_time_ms": 188278.221, "update_time_ms": 8.658, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5574.14453125, "policy_loss": 0.0005272022681310773, "vf_loss": 5574.14453125, "vf_explained_var": 0.1438683122396469, "vf_preds": 121.38139343261719, "kl": 0.006178526673465967, "entropy": -0.3020080626010895, "entropy_coeff": 0.0, "advantages": 7.665045814064797e-06, "rewards": 1.8805773258209229}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2873000, "episodes_total": 2873, "training_iteration": 169, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_09-40-52", "timestamp": 1599385252, "time_this_iter_s": 2319.597658395767, "time_total_s": 375856.05653238297, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc0a96a8>", "on_episode_step": ".on_episode_step at 0x7f3afc0a9598>", "on_episode_end": ".on_episode_end at 0x7f3afc0a9620>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc0a9730>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 375856.05653238297, "timesteps_since_restore": 2873000, "iterations_since_restore": 169, "perf": {"cpu_util_percent": 87.82478813559321, "ram_util_percent": 30.689255447941886}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 142932.80295707195, "episode_reward_min": 139803.02670968787, "episode_reward_mean": 141079.52904769615, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 624.8818127232084}, "policy_reward_mean": {"av": 449.87094721841885}, "custom_metrics": {"avg_speed_mean": 4.927543860715145, "avg_speed_min": 4.833508281528948, "avg_speed_max": 5.007642169957074, "avg_speed_avs_mean": 4.915383526852536, "avg_speed_avs_min": 4.817095492805812, "avg_speed_avs_max": 4.995642160562853, "avg_headway_avs_mean": 9.843974559566375, "avg_headway_avs_min": 9.188445071230188, "avg_headway_avs_max": 10.76117839938169, "std_headway_avs_mean": 8.273903626638544, "std_headway_avs_min": 7.598908336162687, "std_headway_avs_max": 8.709787638189193, "avg_accel_avs_mean": 0.24858033021401912, "avg_accel_avs_min": 0.2375088450691289, "avg_accel_avs_max": 0.2564371111041489, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.34535980820241, "avg_mpg_per_veh_min": 30.652440113036356, "avg_mpg_per_veh_max": 32.000687398740375, "num_cars_mean": 938.5397400000002, "num_cars_min": 928.659, "num_cars_max": 947.574, "avg_mpg_mean": 31.192140392412025, "avg_mpg_min": 30.522180280354334, "avg_mpg_max": 31.84424504609391}, "sampler_perf": {"mean_env_wait_ms": 1577.6917465556678, "mean_processing_ms": 431.06080712321506, "mean_inference_ms": 1.856836089054655}, "off_policy_estimator": {}, "info": {"num_steps_trained": 206759552, "num_steps_sampled": 2890000, "sample_time_ms": 2074316.51, "load_time_ms": 461.179, "grad_time_ms": 189502.65, "update_time_ms": 8.412, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5484.29736328125, "policy_loss": 0.0006792954518459737, "vf_loss": 5484.296875, "vf_explained_var": 0.13930028676986694, "vf_preds": 118.11775207519531, "kl": 0.0064935386180877686, "entropy": -0.3514542877674103, "entropy_coeff": 0.0, "advantages": 2.7793876142823137e-06, "rewards": 1.8719406127929688}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2890000, "episodes_total": 2890, "training_iteration": 170, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_10-18-51", "timestamp": 1599387531, "time_this_iter_s": 2278.5584168434143, "time_total_s": 378134.6149492264, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3af4f76950>", "on_episode_step": ".on_episode_step at 0x7f3af4f768c8>", "on_episode_end": ".on_episode_end at 0x7f3af4f76840>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3af4f76048>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 378134.6149492264, "timesteps_since_restore": 2890000, "iterations_since_restore": 170, "perf": {"cpu_util_percent": 88.02459167950693, "ram_util_percent": 30.739383667180277}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 142071.3206608528, "episode_reward_min": 139803.02670968787, "episode_reward_mean": 140969.11865179462, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 624.8818127232084}, "policy_reward_mean": {"av": 449.8488006248035}, "custom_metrics": {"avg_speed_mean": 4.922621475802617, "avg_speed_min": 4.833508281528948, "avg_speed_max": 4.996601265081051, "avg_speed_avs_mean": 4.910060191944759, "avg_speed_avs_min": 4.817095492805812, "avg_speed_avs_max": 4.984136569881391, "avg_headway_avs_mean": 9.870710695111349, "avg_headway_avs_min": 9.10048406745011, "avg_headway_avs_max": 10.76117839938169, "std_headway_avs_mean": 8.281193848618141, "std_headway_avs_min": 7.704510534426904, "std_headway_avs_max": 8.709787638189193, "avg_accel_avs_mean": 0.24597085071368782, "avg_accel_avs_min": 0.2326745185472656, "avg_accel_avs_max": 0.25396621510526196, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.345188077564767, "avg_mpg_per_veh_min": 30.652440113036356, "avg_mpg_per_veh_max": 32.000687398740375, "num_cars_mean": 938.7938400000002, "num_cars_min": 928.659, "num_cars_max": 947.574, "avg_mpg_mean": 31.19197006157383, "avg_mpg_min": 30.522180280354334, "avg_mpg_max": 31.84424504609391}, "sampler_perf": {"mean_env_wait_ms": 1578.2161631248593, "mean_processing_ms": 430.9138988102747, "mean_inference_ms": 1.8571787142095706}, "off_policy_estimator": {}, "info": {"num_steps_trained": 207998080, "num_steps_sampled": 2907000, "sample_time_ms": 2077690.04, "load_time_ms": 456.687, "grad_time_ms": 190838.68, "update_time_ms": 8.491, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5508.9716796875, "policy_loss": 0.0005664622294716537, "vf_loss": 5508.970703125, "vf_explained_var": 0.1387011706829071, "vf_preds": 118.51068115234375, "kl": 0.006135901901870966, "entropy": -0.36027976870536804, "entropy_coeff": 0.0, "advantages": 6.696338004985591e-06, "rewards": 1.873444676399231}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2907000, "episodes_total": 2907, "training_iteration": 171, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_10-56-57", "timestamp": 1599389817, "time_this_iter_s": 2286.637873649597, "time_total_s": 380421.252822876, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3af4fa9488>", "on_episode_step": ".on_episode_step at 0x7f3af4fa9510>", "on_episode_end": ".on_episode_end at 0x7f3afc084048>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc084bf8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 380421.252822876, "timesteps_since_restore": 2907000, "iterations_since_restore": 171, "perf": {"cpu_util_percent": 87.86760822843108, "ram_util_percent": 30.760362296591957}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 142071.3206608528, "episode_reward_min": 139934.6916298782, "episode_reward_mean": 141087.31441389013, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 624.8818127232084}, "policy_reward_mean": {"av": 450.0823505084701}, "custom_metrics": {"avg_speed_mean": 4.928298385095597, "avg_speed_min": 4.833508281528948, "avg_speed_max": 4.996601265081051, "avg_speed_avs_mean": 4.9161362195889104, "avg_speed_avs_min": 4.817095492805812, "avg_speed_avs_max": 4.984136569881391, "avg_headway_avs_mean": 9.909285586244103, "avg_headway_avs_min": 9.10048406745011, "avg_headway_avs_max": 10.76117839938169, "std_headway_avs_mean": 8.293131651081048, "std_headway_avs_min": 7.704510534426904, "std_headway_avs_max": 8.709787638189193, "avg_accel_avs_mean": 0.2428399983945687, "avg_accel_avs_min": 0.2269936085487467, "avg_accel_avs_max": 0.25396621510526196, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.41797987998799, "avg_mpg_per_veh_min": 30.769235273709246, "avg_mpg_per_veh_max": 32.01087337354766, "num_cars_mean": 938.33454, "num_cars_min": 928.659, "num_cars_max": 947.574, "avg_mpg_mean": 31.26236602012546, "avg_mpg_min": 30.616040996267817, "avg_mpg_max": 31.860658497763275}, "sampler_perf": {"mean_env_wait_ms": 1578.8150487493613, "mean_processing_ms": 430.6667874029845, "mean_inference_ms": 1.857622401589333}, "off_policy_estimator": {}, "info": {"num_steps_trained": 209233280, "num_steps_sampled": 2924000, "sample_time_ms": 2077936.74, "load_time_ms": 455.925, "grad_time_ms": 191672.044, "update_time_ms": 8.615, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5355.916015625, "policy_loss": 0.0005911596235819161, "vf_loss": 5355.91552734375, "vf_explained_var": 0.15401425957679749, "vf_preds": 120.97763061523438, "kl": 0.00643925229087472, "entropy": -0.40888890624046326, "entropy_coeff": 0.0, "advantages": 1.116905241360655e-05, "rewards": 1.878345012664795}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2924000, "episodes_total": 2924, "training_iteration": 172, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_11-34-28", "timestamp": 1599392068, "time_this_iter_s": 2250.093092918396, "time_total_s": 382671.3459157944, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3af4f63d08>", "on_episode_step": ".on_episode_step at 0x7f3af4f63ea0>", "on_episode_end": ".on_episode_end at 0x7f3af4f63f28>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3af4f63a60>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 382671.3459157944, "timesteps_since_restore": 2924000, "iterations_since_restore": 172, "perf": {"cpu_util_percent": 87.34229088639201, "ram_util_percent": 30.899375780274653}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 142071.3206608528, "episode_reward_min": 138979.3322366201, "episode_reward_mean": 140920.55084585428, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 624.8818127232084}, "policy_reward_mean": {"av": 450.05285783678556}, "custom_metrics": {"avg_speed_mean": 4.91967565432879, "avg_speed_min": 4.796638487030082, "avg_speed_max": 4.996601265081051, "avg_speed_avs_mean": 4.907254353384636, "avg_speed_avs_min": 4.781953663799236, "avg_speed_avs_max": 4.984136569881391, "avg_headway_avs_mean": 9.93045745161438, "avg_headway_avs_min": 9.10048406745011, "avg_headway_avs_max": 10.76117839938169, "std_headway_avs_mean": 8.274886099482526, "std_headway_avs_min": 7.704510534426904, "std_headway_avs_max": 8.700834426387232, "avg_accel_avs_mean": 0.23987416707388065, "avg_accel_avs_min": 0.2269936085487467, "avg_accel_avs_max": 0.25396621510526196, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.422779589298344, "avg_mpg_per_veh_min": 30.757478466727473, "avg_mpg_per_veh_max": 32.01087337354766, "num_cars_mean": 938.8766299999999, "num_cars_min": 928.659, "num_cars_max": 948.778, "avg_mpg_mean": 31.267266848288063, "avg_mpg_min": 30.599002503128133, "avg_mpg_max": 31.860658497763275}, "sampler_perf": {"mean_env_wait_ms": 1579.4555879784382, "mean_processing_ms": 430.329131779023, "mean_inference_ms": 1.8581239035239108}, "off_policy_estimator": {}, "info": {"num_steps_trained": 210477952, "num_steps_sampled": 2941000, "sample_time_ms": 2076993.035, "load_time_ms": 454.57, "grad_time_ms": 192362.995, "update_time_ms": 8.672, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5391.685546875, "policy_loss": 0.0007314331014640629, "vf_loss": 5391.68505859375, "vf_explained_var": 0.14375324547290802, "vf_preds": 119.52327728271484, "kl": 0.0068270559422671795, "entropy": -0.4426737427711487, "entropy_coeff": 0.0, "advantages": -3.89950992030208e-06, "rewards": 1.8415874242782593}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2941000, "episodes_total": 2941, "training_iteration": 173, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_12-11-46", "timestamp": 1599394306, "time_this_iter_s": 2238.350563287735, "time_total_s": 384909.6964790821, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3af4fa9620>", "on_episode_step": ".on_episode_step at 0x7f3af4fa9598>", "on_episode_end": ".on_episode_end at 0x7f3af4fa90d0>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3af4fa9158>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 384909.6964790821, "timesteps_since_restore": 2941000, "iterations_since_restore": 173, "perf": {"cpu_util_percent": 87.25619316400126, "ram_util_percent": 30.72715584822829}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 142071.3206608528, "episode_reward_min": 138340.00701932138, "episode_reward_mean": 140629.31216921305, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 624.8818127232084}, "policy_reward_mean": {"av": 449.75474021112024}, "custom_metrics": {"avg_speed_mean": 4.905201983560472, "avg_speed_min": 4.796638487030082, "avg_speed_max": 4.996601265081051, "avg_speed_avs_mean": 4.89206966222692, "avg_speed_avs_min": 4.781953663799236, "avg_speed_avs_max": 4.984136569881391, "avg_headway_avs_mean": 9.947248418568831, "avg_headway_avs_min": 9.10048406745011, "avg_headway_avs_max": 10.82536400064061, "std_headway_avs_mean": 8.26356661876168, "std_headway_avs_min": 7.704510534426904, "std_headway_avs_max": 8.700834426387232, "avg_accel_avs_mean": 0.2366721086152375, "avg_accel_avs_min": 0.22608565008566386, "avg_accel_avs_max": 0.2526747884414843, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.377355484474773, "avg_mpg_per_veh_min": 30.664907571733334, "avg_mpg_per_veh_max": 32.01087337354766, "num_cars_mean": 939.69083, "num_cars_min": 928.659, "num_cars_max": 948.778, "avg_mpg_mean": 31.221627525153576, "avg_mpg_min": 30.539292427363623, "avg_mpg_max": 31.860658497763275}, "sampler_perf": {"mean_env_wait_ms": 1580.0808678749981, "mean_processing_ms": 429.95187445193545, "mean_inference_ms": 1.8585875051849365}, "off_policy_estimator": {}, "info": {"num_steps_trained": 211723008, "num_steps_sampled": 2958000, "sample_time_ms": 2080107.403, "load_time_ms": 452.922, "grad_time_ms": 192470.407, "update_time_ms": 8.631, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5002.20263671875, "policy_loss": 0.0005655035492964089, "vf_loss": 5002.20166015625, "vf_explained_var": 0.14720183610916138, "vf_preds": 120.92485046386719, "kl": 0.0071249171160161495, "entropy": -0.4489394426345825, "entropy_coeff": 0.0, "advantages": -4.526297743723262e-06, "rewards": 1.8363054990768433}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2958000, "episodes_total": 2958, "training_iteration": 174, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_12-49-21", "timestamp": 1599396561, "time_this_iter_s": 2255.0973587036133, "time_total_s": 387164.7938377857, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3af4f63bf8>", "on_episode_step": ".on_episode_step at 0x7f3af4f63c80>", "on_episode_end": ".on_episode_end at 0x7f3af4f63158>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3af4f631e0>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 387164.7938377857, "timesteps_since_restore": 2958000, "iterations_since_restore": 174, "perf": {"cpu_util_percent": 88.28437110834372, "ram_util_percent": 30.714663760896638}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 141978.225673836, "episode_reward_min": 138340.00701932138, "episode_reward_mean": 140447.21429274848, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 623.3587262720063}, "policy_reward_mean": {"av": 449.5173930762658}, "custom_metrics": {"avg_speed_mean": 4.890876464231609, "avg_speed_min": 4.78926253667255, "avg_speed_max": 4.996601265081051, "avg_speed_avs_mean": 4.877532776604898, "avg_speed_avs_min": 4.773884374397401, "avg_speed_avs_max": 4.984136569881391, "avg_headway_avs_mean": 9.907459831044049, "avg_headway_avs_min": 9.045807988200359, "avg_headway_avs_max": 10.82536400064061, "std_headway_avs_mean": 8.21590221275251, "std_headway_avs_min": 7.542190415000937, "std_headway_avs_max": 8.656035967389496, "avg_accel_avs_mean": 0.23301296790968998, "avg_accel_avs_min": 0.22091042429664232, "avg_accel_avs_max": 0.24382644506172904, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.369031709581567, "avg_mpg_per_veh_min": 30.664907571733334, "avg_mpg_per_veh_max": 32.01087337354766, "num_cars_mean": 940.94625, "num_cars_min": 931.819, "num_cars_max": 951.935, "avg_mpg_mean": 31.215329838603466, "avg_mpg_min": 30.539292427363623, "avg_mpg_max": 31.860658497763275}, "sampler_perf": {"mean_env_wait_ms": 1580.644312978057, "mean_processing_ms": 429.56727375480455, "mean_inference_ms": 1.8589786299942435}, "off_policy_estimator": {}, "info": {"num_steps_trained": 212968960, "num_steps_sampled": 2975000, "sample_time_ms": 2079834.299, "load_time_ms": 451.819, "grad_time_ms": 192531.482, "update_time_ms": 8.513, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4759.85107421875, "policy_loss": 0.0006436523399315774, "vf_loss": 4759.85009765625, "vf_explained_var": 0.14996211230754852, "vf_preds": 123.72511291503906, "kl": 0.006617221515625715, "entropy": -0.4403502345085144, "entropy_coeff": 0.0, "advantages": -5.324069661583053e-06, "rewards": 1.847108244895935}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2975000, "episodes_total": 2975, "training_iteration": 175, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_13-26-33", "timestamp": 1599398793, "time_this_iter_s": 2231.959516763687, "time_total_s": 389396.7533545494, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3af4f630d0>", "on_episode_step": ".on_episode_step at 0x7f3af4f632f0>", "on_episode_end": ".on_episode_end at 0x7f3af4f63378>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3af4f63048>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 389396.7533545494, "timesteps_since_restore": 2975000, "iterations_since_restore": 175, "perf": {"cpu_util_percent": 87.25973576596412, "ram_util_percent": 30.740295690468702}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 141978.225673836, "episode_reward_min": 138340.00701932138, "episode_reward_mean": 140446.8568664395, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 624.9954333479574}, "policy_reward_mean": {"av": 449.5881970179567}, "custom_metrics": {"avg_speed_mean": 4.886343583832103, "avg_speed_min": 4.78926253667255, "avg_speed_max": 4.996601265081051, "avg_speed_avs_mean": 4.873507587050299, "avg_speed_avs_min": 4.773884374397401, "avg_speed_avs_max": 4.984136569881391, "avg_headway_avs_mean": 9.888370473675986, "avg_headway_avs_min": 9.045807988200359, "avg_headway_avs_max": 10.82536400064061, "std_headway_avs_mean": 8.18035265898572, "std_headway_avs_min": 7.542190415000937, "std_headway_avs_max": 8.656035967389496, "avg_accel_avs_mean": 0.22966695828767647, "avg_accel_avs_min": 0.21621832066959304, "avg_accel_avs_max": 0.2405025757476898, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.413941358004045, "avg_mpg_per_veh_min": 30.664907571733334, "avg_mpg_per_veh_max": 32.13566790175034, "num_cars_mean": 941.4179500000001, "num_cars_min": 931.819, "num_cars_max": 951.935, "avg_mpg_mean": 31.25862345844428, "avg_mpg_min": 30.539292427363623, "avg_mpg_max": 31.97107687209673}, "sampler_perf": {"mean_env_wait_ms": 1581.11990899572, "mean_processing_ms": 429.2587926223965, "mean_inference_ms": 1.8592856768292405}, "off_policy_estimator": {}, "info": {"num_steps_trained": 214210560, "num_steps_sampled": 2992000, "sample_time_ms": 2080264.372, "load_time_ms": 447.8, "grad_time_ms": 192131.208, "update_time_ms": 8.197, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 4990.91748046875, "policy_loss": 0.0005192681564949453, "vf_loss": 4990.91650390625, "vf_explained_var": 0.16016939282417297, "vf_preds": 126.68424224853516, "kl": 0.006347935181111097, "entropy": -0.44127357006073, "entropy_coeff": 0.0, "advantages": -4.1723542381078005e-06, "rewards": 1.8673443794250488}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 2992000, "episodes_total": 2992, "training_iteration": 176, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_14-04-12", "timestamp": 1599401052, "time_this_iter_s": 2258.844730615616, "time_total_s": 391655.598085165, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5979d8>", "on_episode_step": ".on_episode_step at 0x7f3afc597378>", "on_episode_end": ".on_episode_end at 0x7f3afc597598>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc597a60>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 391655.598085165, "timesteps_since_restore": 2992000, "iterations_since_restore": 176, "perf": {"cpu_util_percent": 89.45725831520049, "ram_util_percent": 30.66319552377992}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 141978.225673836, "episode_reward_min": 138340.00701932138, "episode_reward_mean": 140446.52221506924, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 624.9954333479574}, "policy_reward_mean": {"av": 449.7022900805906}, "custom_metrics": {"avg_speed_mean": 4.883938066620458, "avg_speed_min": 4.78926253667255, "avg_speed_max": 4.975966963588528, "avg_speed_avs_mean": 4.8714840652805425, "avg_speed_avs_min": 4.773884374397401, "avg_speed_avs_max": 4.970281205315894, "avg_headway_avs_mean": 9.914515798357062, "avg_headway_avs_min": 9.045807988200359, "avg_headway_avs_max": 10.82536400064061, "std_headway_avs_mean": 8.17993704849663, "std_headway_avs_min": 7.542190415000937, "std_headway_avs_max": 8.705153395516545, "avg_accel_avs_mean": 0.2261159315138082, "avg_accel_avs_min": 0.2081180698452486, "avg_accel_avs_max": 0.23983689910914724, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.46628488419885, "avg_mpg_per_veh_min": 30.664907571733334, "avg_mpg_per_veh_max": 32.18339171501255, "num_cars_mean": 941.5436, "num_cars_min": 933.113, "num_cars_max": 951.935, "avg_mpg_mean": 31.31112533574715, "avg_mpg_min": 30.539292427363623, "avg_mpg_max": 32.02847704987857}, "sampler_perf": {"mean_env_wait_ms": 1581.4767065080675, "mean_processing_ms": 429.02585547340294, "mean_inference_ms": 1.8594729778144141}, "off_policy_estimator": {}, "info": {"num_steps_trained": 215448832, "num_steps_sampled": 3009000, "sample_time_ms": 2075199.575, "load_time_ms": 446.309, "grad_time_ms": 190679.45, "update_time_ms": 8.112, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5194.15234375, "policy_loss": 0.0006117297452874482, "vf_loss": 5194.1513671875, "vf_explained_var": 0.16386833786964417, "vf_preds": 127.68871307373047, "kl": 0.006267451215535402, "entropy": -0.4689810276031494, "entropy_coeff": 0.0, "advantages": -1.162935859611025e-05, "rewards": 1.874536395072937}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 3009000, "episodes_total": 3009, "training_iteration": 177, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_14-41-18", "timestamp": 1599403278, "time_this_iter_s": 2225.3769278526306, "time_total_s": 393880.97501301765, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3af4fa9598>", "on_episode_step": ".on_episode_step at 0x7f3af4fa98c8>", "on_episode_end": ".on_episode_end at 0x7f3afc3b0400>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc3b0268>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 393880.97501301765, "timesteps_since_restore": 3009000, "iterations_since_restore": 177, "perf": {"cpu_util_percent": 87.50870937204165, "ram_util_percent": 30.732817923635213}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144127.87701623864, "episode_reward_min": 138340.00701932138, "episode_reward_mean": 140798.24566020328, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 624.9954333479574}, "policy_reward_mean": {"av": 450.36703342674497}, "custom_metrics": {"avg_speed_mean": 4.895524737399891, "avg_speed_min": 4.78926253667255, "avg_speed_max": 5.040678136486365, "avg_speed_avs_mean": 4.8838186884334736, "avg_speed_avs_min": 4.773884374397401, "avg_speed_avs_max": 5.033420875037935, "avg_headway_avs_mean": 9.90400260726077, "avg_headway_avs_min": 9.045807988200359, "avg_headway_avs_max": 10.82536400064061, "std_headway_avs_mean": 8.156617674504384, "std_headway_avs_min": 7.542190415000937, "std_headway_avs_max": 8.705153395516545, "avg_accel_avs_mean": 0.22230363984475246, "avg_accel_avs_min": 0.20523730129180648, "avg_accel_avs_max": 0.2371949771174775, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.616522543042684, "avg_mpg_per_veh_min": 30.664907571733334, "avg_mpg_per_veh_max": 32.847472462671654, "num_cars_mean": 941.1169799999999, "num_cars_min": 928.456, "num_cars_max": 951.935, "avg_mpg_mean": 31.460963542932685, "avg_mpg_min": 30.539292427363623, "avg_mpg_max": 32.6807001611227}, "sampler_perf": {"mean_env_wait_ms": 1581.7242127335846, "mean_processing_ms": 428.8967130995023, "mean_inference_ms": 1.8595513621815853}, "off_policy_estimator": {}, "info": {"num_steps_trained": 216679552, "num_steps_sampled": 3026000, "sample_time_ms": 2068415.443, "load_time_ms": 448.925, "grad_time_ms": 188131.822, "update_time_ms": 8.171, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5556.86669921875, "policy_loss": 0.0004341582825873047, "vf_loss": 5556.8662109375, "vf_explained_var": 0.17618806660175323, "vf_preds": 130.18899536132812, "kl": 0.006653794553130865, "entropy": -0.467380166053772, "entropy_coeff": 0.0, "advantages": -5.911082098464249e-06, "rewards": 1.9103108644485474}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 3026000, "episodes_total": 3026, "training_iteration": 178, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_15-18-27", "timestamp": 1599405507, "time_this_iter_s": 2229.6728501319885, "time_total_s": 396110.64786314964, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc597f28>", "on_episode_step": ".on_episode_step at 0x7f3afc597bf8>", "on_episode_end": ".on_episode_end at 0x7f3afc597620>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5976a8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 396110.64786314964, "timesteps_since_restore": 3026000, "iterations_since_restore": 178, "perf": {"cpu_util_percent": 88.55198362720404, "ram_util_percent": 30.76199622166247}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144127.87701623864, "episode_reward_min": 138340.00701932138, "episode_reward_mean": 141217.74676990983, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 624.9954333479574}, "policy_reward_mean": {"av": 451.0020016923538}, "custom_metrics": {"avg_speed_mean": 4.912978706621948, "avg_speed_min": 4.78926253667255, "avg_speed_max": 5.040678136486365, "avg_speed_avs_mean": 4.902305626365005, "avg_speed_avs_min": 4.773884374397401, "avg_speed_avs_max": 5.033420875037935, "avg_headway_avs_mean": 9.924731487021273, "avg_headway_avs_min": 9.045807988200359, "avg_headway_avs_max": 10.648993579953016, "std_headway_avs_mean": 8.161277267578855, "std_headway_avs_min": 7.542190415000937, "std_headway_avs_max": 8.705153395516545, "avg_accel_avs_mean": 0.218863104879287, "avg_accel_avs_min": 0.20523730129180648, "avg_accel_avs_max": 0.2344549190357945, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.77126600394963, "avg_mpg_per_veh_min": 30.664907571733334, "avg_mpg_per_veh_max": 32.847472462671654, "num_cars_mean": 940.0595900000001, "num_cars_min": 928.456, "num_cars_max": 951.935, "avg_mpg_mean": 31.611426072051387, "avg_mpg_min": 30.539292427363623, "avg_mpg_max": 32.6807001611227}, "sampler_perf": {"mean_env_wait_ms": 1581.8648739602816, "mean_processing_ms": 428.8657418844101, "mean_inference_ms": 1.859522317046052}, "off_policy_estimator": {}, "info": {"num_steps_trained": 217912960, "num_steps_sampled": 3043000, "sample_time_ms": 2061403.189, "load_time_ms": 450.21, "grad_time_ms": 185146.088, "update_time_ms": 8.013, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5659.35888671875, "policy_loss": 0.0006032424862496555, "vf_loss": 5659.35888671875, "vf_explained_var": 0.18340401351451874, "vf_preds": 126.38118743896484, "kl": 0.00659506069496274, "entropy": -0.4839978516101837, "entropy_coeff": 0.0, "advantages": 2.6032062123704236e-06, "rewards": 1.890398383140564}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 3043000, "episodes_total": 3043, "training_iteration": 179, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_15-55-27", "timestamp": 1599407727, "time_this_iter_s": 2219.6155428886414, "time_total_s": 398330.2634060383, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc4fc268>", "on_episode_step": ".on_episode_step at 0x7f3afc4fcea0>", "on_episode_end": ".on_episode_end at 0x7f3afc4fc8c8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc4fcd90>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 398330.2634060383, "timesteps_since_restore": 3043000, "iterations_since_restore": 179, "perf": {"cpu_util_percent": 88.19832384566729, "ram_util_percent": 30.86246046805819}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144127.87701623864, "episode_reward_min": 139100.14649623417, "episode_reward_mean": 141740.98331491067, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 626.4684427590497}, "policy_reward_mean": {"av": 451.79288979348695}, "custom_metrics": {"avg_speed_mean": 4.935544658688343, "avg_speed_min": 4.82308729411072, "avg_speed_max": 5.040678136486365, "avg_speed_avs_mean": 4.926289477478082, "avg_speed_avs_min": 4.809415458624384, "avg_speed_avs_max": 5.033420875037935, "avg_headway_avs_mean": 9.948282157823392, "avg_headway_avs_min": 9.291100281116483, "avg_headway_avs_max": 10.648993579953016, "std_headway_avs_mean": 8.180298880411298, "std_headway_avs_min": 7.6467554787602845, "std_headway_avs_max": 8.705153395516545, "avg_accel_avs_mean": 0.21513733823484668, "avg_accel_avs_min": 0.20394831401889751, "avg_accel_avs_max": 0.23375653184021858, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 31.969460163968375, "avg_mpg_per_veh_min": 30.77491294851035, "avg_mpg_per_veh_max": 32.847472462671654, "num_cars_mean": 938.66757, "num_cars_min": 928.456, "num_cars_max": 948.263, "avg_mpg_mean": 31.80547301171348, "avg_mpg_min": 30.622733530311848, "avg_mpg_max": 32.6807001611227}, "sampler_perf": {"mean_env_wait_ms": 1581.9235375354858, "mean_processing_ms": 428.9011709923032, "mean_inference_ms": 1.8594058093853834}, "off_policy_estimator": {}, "info": {"num_steps_trained": 219145600, "num_steps_sampled": 3060000, "sample_time_ms": 2058572.503, "load_time_ms": 448.616, "grad_time_ms": 183702.969, "update_time_ms": 8.001, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5928.0068359375, "policy_loss": 0.0006809842889197171, "vf_loss": 5928.0068359375, "vf_explained_var": 0.177785262465477, "vf_preds": 126.11763763427734, "kl": 0.006641417276114225, "entropy": -0.5211576223373413, "entropy_coeff": 0.0, "advantages": 8.69517225510208e-06, "rewards": 1.8981671333312988}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 3060000, "episodes_total": 3060, "training_iteration": 180, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_16-32-43", "timestamp": 1599409963, "time_this_iter_s": 2235.7257125377655, "time_total_s": 400565.98911857605, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3af4f631e0>", "on_episode_step": ".on_episode_step at 0x7f3af4f63f28>", "on_episode_end": ".on_episode_end at 0x7f3af4fa9598>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3af4fa98c8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 400565.98911857605, "timesteps_since_restore": 3060000, "iterations_since_restore": 180, "perf": {"cpu_util_percent": 88.29224246231156, "ram_util_percent": 30.78253768844221}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144611.3420378099, "episode_reward_min": 140708.6431124439, "episode_reward_mean": 142399.96909170353, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 626.4684427590497}, "policy_reward_mean": {"av": 452.8108912862615}, "custom_metrics": {"avg_speed_mean": 4.965798163965187, "avg_speed_min": 4.873990257403408, "avg_speed_max": 5.087668733406034, "avg_speed_avs_mean": 4.95786346035492, "avg_speed_avs_min": 4.863503458498484, "avg_speed_avs_max": 5.088046365182807, "avg_headway_avs_mean": 9.998775329817835, "avg_headway_avs_min": 9.418910094122806, "avg_headway_avs_max": 10.742633600638797, "std_headway_avs_mean": 8.191041086691037, "std_headway_avs_min": 7.675725667582773, "std_headway_avs_max": 8.705153395516545, "avg_accel_avs_mean": 0.21065482910284153, "avg_accel_avs_min": 0.19590861956594505, "avg_accel_avs_max": 0.22454677118218455, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 32.21198959593085, "avg_mpg_per_veh_min": 31.47384729829091, "avg_mpg_per_veh_max": 33.2876838313862, "num_cars_mean": 936.7235099999999, "num_cars_min": 926.821, "num_cars_max": 944.982, "avg_mpg_mean": 32.043002344665744, "avg_mpg_min": 31.308635377469585, "avg_mpg_max": 33.08096494585041}, "sampler_perf": {"mean_env_wait_ms": 1581.929733920493, "mean_processing_ms": 428.9890383246461, "mean_inference_ms": 1.8592498395770096}, "off_policy_estimator": {}, "info": {"num_steps_trained": 220370048, "num_steps_sampled": 3077000, "sample_time_ms": 2053676.048, "load_time_ms": 449.886, "grad_time_ms": 182193.878, "update_time_ms": 8.009, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 6211.67431640625, "policy_loss": 0.0007127124699763954, "vf_loss": 6211.67333984375, "vf_explained_var": 0.19443385303020477, "vf_preds": 127.53392028808594, "kl": 0.0066685061901807785, "entropy": -0.530046284198761, "entropy_coeff": 0.0, "advantages": 3.919843493349617e-06, "rewards": 1.9336374998092651}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 3077000, "episodes_total": 3077, "training_iteration": 181, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_17-09-45", "timestamp": 1599412185, "time_this_iter_s": 2222.589341878891, "time_total_s": 402788.57846045494, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc4fc730>", "on_episode_step": ".on_episode_step at 0x7f3afc4fcf28>", "on_episode_end": ".on_episode_end at 0x7f3afc4fc158>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc4fc598>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 402788.57846045494, "timesteps_since_restore": 3077000, "iterations_since_restore": 181, "perf": {"cpu_util_percent": 88.87210991787744, "ram_util_percent": 30.779343019583067}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144611.3420378099, "episode_reward_min": 140831.20213916508, "episode_reward_mean": 142608.14394173954, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 626.4684427590497}, "policy_reward_mean": {"av": 453.2566631972142}, "custom_metrics": {"avg_speed_mean": 4.975144030623905, "avg_speed_min": 4.873990257403408, "avg_speed_max": 5.087668733406034, "avg_speed_avs_mean": 4.967614487210847, "avg_speed_avs_min": 4.863503458498484, "avg_speed_avs_max": 5.088046365182807, "avg_headway_avs_mean": 10.0464472297068, "avg_headway_avs_min": 9.44548321572326, "avg_headway_avs_max": 10.742633600638797, "std_headway_avs_mean": 8.231351319546278, "std_headway_avs_min": 7.675725667582773, "std_headway_avs_max": 8.705153395516545, "avg_accel_avs_mean": 0.20749864941022533, "avg_accel_avs_min": 0.19590861956594505, "avg_accel_avs_max": 0.22050066990608255, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 32.311887680867365, "avg_mpg_per_veh_min": 31.63390015272247, "avg_mpg_per_veh_max": 33.2876838313862, "num_cars_mean": 936.0295400000001, "num_cars_min": 926.821, "num_cars_max": 943.183, "avg_mpg_mean": 32.14148036499011, "avg_mpg_min": 31.481859313895765, "avg_mpg_max": 33.08096494585041}, "sampler_perf": {"mean_env_wait_ms": 1581.8995070707308, "mean_processing_ms": 429.09316255488807, "mean_inference_ms": 1.8590515056198642}, "off_policy_estimator": {}, "info": {"num_steps_trained": 221603200, "num_steps_sampled": 3094000, "sample_time_ms": 2054532.225, "load_time_ms": 451.497, "grad_time_ms": 182593.228, "update_time_ms": 8.14, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 6052.6982421875, "policy_loss": 0.00074232486076653, "vf_loss": 6052.69775390625, "vf_explained_var": 0.19545112550258636, "vf_preds": 126.01094818115234, "kl": 0.006400662939995527, "entropy": -0.5596734881401062, "entropy_coeff": 0.0, "advantages": -4.676208391174441e-06, "rewards": 1.8963932991027832}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 3094000, "episodes_total": 3094, "training_iteration": 182, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_17-47-28", "timestamp": 1599414448, "time_this_iter_s": 2262.6705079078674, "time_total_s": 405051.2489683628, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc4fc1e0>", "on_episode_step": ".on_episode_step at 0x7f3afc4fc378>", "on_episode_end": ".on_episode_end at 0x7f3afc4fcd90>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc4fcea0>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 405051.2489683628, "timesteps_since_restore": 3094000, "iterations_since_restore": 182, "perf": {"cpu_util_percent": 88.24461681663045, "ram_util_percent": 30.795904436860074}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144611.3420378099, "episode_reward_min": 139206.84239354788, "episode_reward_mean": 142331.56633104765, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 626.4684427590497}, "policy_reward_mean": {"av": 453.0543873537293}, "custom_metrics": {"avg_speed_mean": 4.9649030145742215, "avg_speed_min": 4.814883114317798, "avg_speed_max": 5.087668733406034, "avg_speed_avs_mean": 4.956551072705473, "avg_speed_avs_min": 4.8016310700618865, "avg_speed_avs_max": 5.088046365182807, "avg_headway_avs_mean": 10.114750163451973, "avg_headway_avs_min": 9.44548321572326, "avg_headway_avs_max": 10.88897752992214, "std_headway_avs_mean": 8.299492738447674, "std_headway_avs_min": 7.675725667582773, "std_headway_avs_max": 9.078402687391149, "avg_accel_avs_mean": 0.20627454646585236, "avg_accel_avs_min": 0.19590861956594505, "avg_accel_avs_max": 0.2172965238846018, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 32.26488952109143, "avg_mpg_per_veh_min": 31.34739878101877, "avg_mpg_per_veh_max": 33.2876838313862, "num_cars_mean": 936.3571600000001, "num_cars_min": 926.821, "num_cars_max": 947.483, "avg_mpg_mean": 32.09531249541254, "avg_mpg_min": 31.21231888033195, "avg_mpg_max": 33.08096494585041}, "sampler_perf": {"mean_env_wait_ms": 1581.8678849180233, "mean_processing_ms": 429.2090622236033, "mean_inference_ms": 1.8588699438135992}, "off_policy_estimator": {}, "info": {"num_steps_trained": 222842624, "num_steps_sampled": 3111000, "sample_time_ms": 2056989.203, "load_time_ms": 450.129, "grad_time_ms": 182657.119, "update_time_ms": 7.916, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5777.3154296875, "policy_loss": 0.0008415241609327495, "vf_loss": 5777.314453125, "vf_explained_var": 0.19482262432575226, "vf_preds": 124.6866683959961, "kl": 0.007366555742919445, "entropy": -0.5939825773239136, "entropy_coeff": 0.0, "advantages": -7.096866283973213e-06, "rewards": 1.8544055223464966}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 3111000, "episodes_total": 3111, "training_iteration": 183, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_18-25-12", "timestamp": 1599416712, "time_this_iter_s": 2263.5625698566437, "time_total_s": 407314.81153821945, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc541840>", "on_episode_step": ".on_episode_step at 0x7f3afc5418c8>", "on_episode_end": ".on_episode_end at 0x7f3afc541950>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5419d8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 407314.81153821945, "timesteps_since_restore": 3111000, "iterations_since_restore": 183, "perf": {"cpu_util_percent": 87.5541253101737, "ram_util_percent": 30.92552729528536}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144611.3420378099, "episode_reward_min": 139206.84239354788, "episode_reward_mean": 142000.28902560307, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 626.4684427590497}, "policy_reward_mean": {"av": 452.6049882884015}, "custom_metrics": {"avg_speed_mean": 4.954600776415899, "avg_speed_min": 4.814883114317798, "avg_speed_max": 5.087668733406034, "avg_speed_avs_mean": 4.945876857659176, "avg_speed_avs_min": 4.8016310700618865, "avg_speed_avs_max": 5.088046365182807, "avg_headway_avs_mean": 10.197808302889817, "avg_headway_avs_min": 9.644029263824253, "avg_headway_avs_max": 10.88897752992214, "std_headway_avs_mean": 8.372834583491715, "std_headway_avs_min": 7.779894790555139, "std_headway_avs_max": 9.078402687391149, "avg_accel_avs_mean": 0.20599121903292006, "avg_accel_avs_min": 0.19590861956594505, "avg_accel_avs_max": 0.2172965238846018, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 32.18469315842008, "avg_mpg_per_veh_min": 31.269545101075682, "avg_mpg_per_veh_max": 33.2876838313862, "num_cars_mean": 936.4896800000001, "num_cars_min": 926.821, "num_cars_max": 947.483, "avg_mpg_mean": 32.014985789182866, "avg_mpg_min": 31.110833429936022, "avg_mpg_max": 33.08096494585041}, "sampler_perf": {"mean_env_wait_ms": 1581.8319778108812, "mean_processing_ms": 429.325139070268, "mean_inference_ms": 1.8586833611539229}, "off_policy_estimator": {}, "info": {"num_steps_trained": 224075392, "num_steps_sampled": 3128000, "sample_time_ms": 2055118.777, "load_time_ms": 451.782, "grad_time_ms": 182926.155, "update_time_ms": 8.031, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 6073.03564453125, "policy_loss": 0.0006960708415135741, "vf_loss": 6073.03466796875, "vf_explained_var": 0.19284601509571075, "vf_preds": 123.96234893798828, "kl": 0.007040095515549183, "entropy": -0.5638270378112793, "entropy_coeff": 0.0, "advantages": -3.5652005863084923e-06, "rewards": 1.8789705038070679}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 3128000, "episodes_total": 3128, "training_iteration": 184, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_19-02-31", "timestamp": 1599418951, "time_this_iter_s": 2239.104259252548, "time_total_s": 409553.915797472, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc4fcf28>", "on_episode_step": ".on_episode_step at 0x7f3afc4fc158>", "on_episode_end": ".on_episode_end at 0x7f3afc4fc598>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc4fcd90>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 409553.915797472, "timesteps_since_restore": 3128000, "iterations_since_restore": 184, "perf": {"cpu_util_percent": 87.9728127939793, "ram_util_percent": 30.90200689871433}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144611.3420378099, "episode_reward_min": 139206.84239354788, "episode_reward_mean": 141873.39612027342, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 626.4684427590497}, "policy_reward_mean": {"av": 452.46012284817385}, "custom_metrics": {"avg_speed_mean": 4.948318534526711, "avg_speed_min": 4.814883114317798, "avg_speed_max": 5.087668733406034, "avg_speed_avs_mean": 4.9396199026302385, "avg_speed_avs_min": 4.8016310700618865, "avg_speed_avs_max": 5.088046365182807, "avg_headway_avs_mean": 10.222856415791991, "avg_headway_avs_min": 9.644029263824253, "avg_headway_avs_max": 10.88897752992214, "std_headway_avs_mean": 8.38157676458684, "std_headway_avs_min": 7.779894790555139, "std_headway_avs_max": 9.078402687391149, "avg_accel_avs_mean": 0.20492348020108245, "avg_accel_avs_min": 0.19590861956594505, "avg_accel_avs_max": 0.2161409591878806, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 32.182057579860846, "avg_mpg_per_veh_min": 31.269545101075682, "avg_mpg_per_veh_max": 33.2876838313862, "num_cars_mean": 936.7899699999999, "num_cars_min": 926.821, "num_cars_max": 947.483, "avg_mpg_mean": 32.013011213631266, "avg_mpg_min": 31.110833429936022, "avg_mpg_max": 33.08096494585041}, "sampler_perf": {"mean_env_wait_ms": 1581.7890452397644, "mean_processing_ms": 429.43951860415734, "mean_inference_ms": 1.8584967311117666}, "off_policy_estimator": {}, "info": {"num_steps_trained": 225309824, "num_steps_sampled": 3145000, "sample_time_ms": 2053562.334, "load_time_ms": 452.806, "grad_time_ms": 183397.25, "update_time_ms": 7.894, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5984.80322265625, "policy_loss": 0.0006604564259760082, "vf_loss": 5984.802734375, "vf_explained_var": 0.1920497864484787, "vf_preds": 121.05272674560547, "kl": 0.007062060758471489, "entropy": -0.5660228133201599, "entropy_coeff": 0.0, "advantages": 5.957025223324308e-06, "rewards": 1.8805632591247559}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 3145000, "episodes_total": 3145, "training_iteration": 185, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_19-39-32", "timestamp": 1599421172, "time_this_iter_s": 2221.105535030365, "time_total_s": 411775.02133250237, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3b73421488>", "on_episode_step": ".on_episode_step at 0x7f3b73421048>", "on_episode_end": ".on_episode_end at 0x7f3afc672e18>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc672bf8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 411775.02133250237, "timesteps_since_restore": 3145000, "iterations_since_restore": 185, "perf": {"cpu_util_percent": 87.28631479140327, "ram_util_percent": 30.901769911504424}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144453.8485507121, "episode_reward_min": 139206.84239354788, "episode_reward_mean": 141907.92314720704, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 623.7937583050232}, "policy_reward_mean": {"av": 452.67129141984435}, "custom_metrics": {"avg_speed_mean": 4.9498036469554485, "avg_speed_min": 4.814883114317798, "avg_speed_max": 5.087668733406034, "avg_speed_avs_mean": 4.941227778985798, "avg_speed_avs_min": 4.8016310700618865, "avg_speed_avs_max": 5.088046365182807, "avg_headway_avs_mean": 10.259829324734959, "avg_headway_avs_min": 9.644029263824253, "avg_headway_avs_max": 10.88897752992214, "std_headway_avs_mean": 8.391294425182643, "std_headway_avs_min": 7.775815310389579, "std_headway_avs_max": 9.078402687391149, "avg_accel_avs_mean": 0.2034653811673421, "avg_accel_avs_min": 0.19343526677311124, "avg_accel_avs_max": 0.2161409591878806, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 32.23000072339305, "avg_mpg_per_veh_min": 31.269545101075682, "avg_mpg_per_veh_max": 33.2876838313862, "num_cars_mean": 936.62415, "num_cars_min": 926.821, "num_cars_max": 947.483, "avg_mpg_mean": 32.06135921082918, "avg_mpg_min": 31.110833429936022, "avg_mpg_max": 33.08096494585041}, "sampler_perf": {"mean_env_wait_ms": 1581.7450152353754, "mean_processing_ms": 429.5528622196243, "mean_inference_ms": 1.8583183795255778}, "off_policy_estimator": {}, "info": {"num_steps_trained": 226538880, "num_steps_sampled": 3162000, "sample_time_ms": 2050930.065, "load_time_ms": 459.855, "grad_time_ms": 183743.887, "update_time_ms": 7.89, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 6081.701171875, "policy_loss": 0.0007748623611405492, "vf_loss": 6081.69970703125, "vf_explained_var": 0.1866673231124878, "vf_preds": 119.52119445800781, "kl": 0.00714586628600955, "entropy": -0.5782238245010376, "entropy_coeff": 0.0, "advantages": -5.529086138267303e-06, "rewards": 1.9086146354675293}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 3162000, "episodes_total": 3162, "training_iteration": 186, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_20-16-48", "timestamp": 1599423408, "time_this_iter_s": 2235.99884057045, "time_total_s": 414011.0201730728, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc672268>", "on_episode_step": ".on_episode_step at 0x7f3afc672a60>", "on_episode_end": ".on_episode_end at 0x7f3b004c89d8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc3b01e0>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 414011.0201730728, "timesteps_since_restore": 3162000, "iterations_since_restore": 186, "perf": {"cpu_util_percent": 88.30543171114601, "ram_util_percent": 30.771742543171115}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144731.47820032525, "episode_reward_min": 139206.84239354788, "episode_reward_mean": 141932.65354569518, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 623.7937583050232}, "policy_reward_mean": {"av": 452.6346702353386}, "custom_metrics": {"avg_speed_mean": 4.949147817878413, "avg_speed_min": 4.814883114317798, "avg_speed_max": 5.082070039152803, "avg_speed_avs_mean": 4.940840615583149, "avg_speed_avs_min": 4.8016310700618865, "avg_speed_avs_max": 5.078516207344268, "avg_headway_avs_mean": 10.29106066374226, "avg_headway_avs_min": 9.644029263824253, "avg_headway_avs_max": 11.009427511561938, "std_headway_avs_mean": 8.411758110304685, "std_headway_avs_min": 7.775815310389579, "std_headway_avs_max": 9.078402687391149, "avg_accel_avs_mean": 0.20178694206758652, "avg_accel_avs_min": 0.18464743552315394, "avg_accel_avs_max": 0.2161409591878806, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 32.285702659131694, "avg_mpg_per_veh_min": 31.269545101075682, "avg_mpg_per_veh_max": 33.43863112398615, "num_cars_mean": 936.5276299999998, "num_cars_min": 925.928, "num_cars_max": 947.483, "avg_mpg_mean": 32.117196899736534, "avg_mpg_min": 31.110833429936022, "avg_mpg_max": 33.243125229249735}, "sampler_perf": {"mean_env_wait_ms": 1581.6822533168538, "mean_processing_ms": 429.6640392206011, "mean_inference_ms": 1.858115285661976}, "off_policy_estimator": {}, "info": {"num_steps_trained": 227762816, "num_steps_sampled": 3179000, "sample_time_ms": 2048652.037, "load_time_ms": 458.553, "grad_time_ms": 184365.203, "update_time_ms": 7.841, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5899.51123046875, "policy_loss": 0.0008305442170239985, "vf_loss": 5899.5107421875, "vf_explained_var": 0.20109440386295319, "vf_preds": 119.96074676513672, "kl": 0.006459775380790234, "entropy": -0.5987359881401062, "entropy_coeff": 0.0, "advantages": 5.1859283303201664e-06, "rewards": 1.9338860511779785}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 3179000, "episodes_total": 3179, "training_iteration": 187, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_20-53-37", "timestamp": 1599425617, "time_this_iter_s": 2208.780713558197, "time_total_s": 416219.800886631, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc570c80>", "on_episode_step": ".on_episode_step at 0x7f3afc570bf8>", "on_episode_end": ".on_episode_end at 0x7f3afc570e18>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc570048>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 416219.800886631, "timesteps_since_restore": 3179000, "iterations_since_restore": 187, "perf": {"cpu_util_percent": 87.63293070565797, "ram_util_percent": 30.80956770502225}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 145951.642291145, "episode_reward_min": 139206.84239354788, "episode_reward_mean": 142523.37829288075, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 624.136875497511}, "policy_reward_mean": {"av": 453.3907373719761}, "custom_metrics": {"avg_speed_mean": 4.9751554157652, "avg_speed_min": 4.814883114317798, "avg_speed_max": 5.167655353508848, "avg_speed_avs_mean": 4.968319195170076, "avg_speed_avs_min": 4.8016310700618865, "avg_speed_avs_max": 5.172621735862994, "avg_headway_avs_mean": 10.261889069347049, "avg_headway_avs_min": 9.136836664049085, "avg_headway_avs_max": 11.009427511561938, "std_headway_avs_mean": 8.377715625437746, "std_headway_avs_min": 7.775815310389579, "std_headway_avs_max": 9.078402687391149, "avg_accel_avs_mean": 0.20027562549954184, "avg_accel_avs_min": 0.18464743552315394, "avg_accel_avs_max": 0.2161409591878806, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 32.46124186092814, "avg_mpg_per_veh_min": 31.269545101075682, "avg_mpg_per_veh_max": 33.67751502333311, "num_cars_mean": 935.2497500000001, "num_cars_min": 921.104, "num_cars_max": 947.483, "avg_mpg_mean": 32.286066816757, "avg_mpg_min": 31.110833429936022, "avg_mpg_max": 33.43499898677152}, "sampler_perf": {"mean_env_wait_ms": 1581.6015717218013, "mean_processing_ms": 429.77268420013485, "mean_inference_ms": 1.8579047304184095}, "off_policy_estimator": {}, "info": {"num_steps_trained": 228983808, "num_steps_sampled": 3196000, "sample_time_ms": 2047058.285, "load_time_ms": 455.862, "grad_time_ms": 184674.851, "update_time_ms": 8.008, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5774.09814453125, "policy_loss": 0.0006520124152302742, "vf_loss": 5774.09765625, "vf_explained_var": 0.238277405500412, "vf_preds": 119.39840698242188, "kl": 0.00709976302459836, "entropy": -0.571887731552124, "entropy_coeff": 0.0, "advantages": 5.478797220348497e-07, "rewards": 1.9575138092041016}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 3196000, "episodes_total": 3196, "training_iteration": 188, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_21-30-34", "timestamp": 1599427834, "time_this_iter_s": 2216.8190500736237, "time_total_s": 418436.61993670464, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc3b09d8>", "on_episode_step": ".on_episode_step at 0x7f3afc3b0e18>", "on_episode_end": ".on_episode_end at 0x7f3afc3b0d90>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc3b02f0>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 418436.61993670464, "timesteps_since_restore": 3196000, "iterations_since_restore": 188, "perf": {"cpu_util_percent": 88.25974025974025, "ram_util_percent": 30.821539436173584}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 145951.642291145, "episode_reward_min": 140070.70799486808, "episode_reward_mean": 143261.4129774449, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 625.2329250873341}, "policy_reward_mean": {"av": 454.36540747683114}, "custom_metrics": {"avg_speed_mean": 5.0061264515826664, "avg_speed_min": 4.870432883396306, "avg_speed_max": 5.167655353508848, "avg_speed_avs_mean": 5.000927339629727, "avg_speed_avs_min": 4.860636683820261, "avg_speed_avs_max": 5.172621735862994, "avg_headway_avs_mean": 10.245621472227294, "avg_headway_avs_min": 9.136836664049085, "avg_headway_avs_max": 11.009427511561938, "std_headway_avs_mean": 8.311608255248593, "std_headway_avs_min": 7.775815310389579, "std_headway_avs_max": 8.862976517638465, "avg_accel_avs_mean": 0.1975554577907059, "avg_accel_avs_min": 0.18464743552315394, "avg_accel_avs_max": 0.21358877073826985, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 32.709662281396625, "avg_mpg_per_veh_min": 31.608489611834408, "avg_mpg_per_veh_max": 33.67751502333311, "num_cars_mean": 933.5378099999999, "num_cars_min": 921.104, "num_cars_max": 943.773, "avg_mpg_mean": 32.526972910770134, "avg_mpg_min": 31.451577637296324, "avg_mpg_max": 33.43499898677152}, "sampler_perf": {"mean_env_wait_ms": 1581.487962998102, "mean_processing_ms": 429.87871406690897, "mean_inference_ms": 1.8576428704120627}, "off_policy_estimator": {}, "info": {"num_steps_trained": 230205824, "num_steps_sampled": 3213000, "sample_time_ms": 2044433.108, "load_time_ms": 455.112, "grad_time_ms": 185276.273, "update_time_ms": 8.174, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 6070.06201171875, "policy_loss": 0.0008671159157529473, "vf_loss": 6070.0615234375, "vf_explained_var": 0.23713770508766174, "vf_preds": 119.2344970703125, "kl": 0.0063339946791529655, "entropy": -0.5727259516716003, "entropy_coeff": 0.0, "advantages": -1.1827106391137931e-05, "rewards": 1.9414482116699219}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 3213000, "episodes_total": 3213, "training_iteration": 189, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_22-07-13", "timestamp": 1599430033, "time_this_iter_s": 2199.390463113785, "time_total_s": 420636.0103998184, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc570f28>", "on_episode_step": ".on_episode_step at 0x7f3afc570378>", "on_episode_end": ".on_episode_end at 0x7f3afc570ea0>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc570400>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 420636.0103998184, "timesteps_since_restore": 3213000, "iterations_since_restore": 189, "perf": {"cpu_util_percent": 87.19074369613789, "ram_util_percent": 30.902266198531755}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 145951.642291145, "episode_reward_min": 140471.51455755538, "episode_reward_mean": 143568.46044638462, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 625.2329250873341}, "policy_reward_mean": {"av": 454.9352317839679}, "custom_metrics": {"avg_speed_mean": 5.018687226369056, "avg_speed_min": 4.870432883396306, "avg_speed_max": 5.167655353508848, "avg_speed_avs_mean": 5.013595162625305, "avg_speed_avs_min": 4.860636683820261, "avg_speed_avs_max": 5.172621735862994, "avg_headway_avs_mean": 10.265694580053347, "avg_headway_avs_min": 9.136836664049085, "avg_headway_avs_max": 11.009427511561938, "std_headway_avs_mean": 8.255328017699592, "std_headway_avs_min": 7.775815310389579, "std_headway_avs_max": 8.677664571899028, "avg_accel_avs_mean": 0.1948566086076708, "avg_accel_avs_min": 0.18464743552315394, "avg_accel_avs_max": 0.211104134107124, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 32.85512637226901, "avg_mpg_per_veh_min": 31.64546016624095, "avg_mpg_per_veh_max": 33.67751502333311, "num_cars_mean": 932.83136, "num_cars_min": 921.104, "num_cars_max": 943.773, "avg_mpg_mean": 32.66818317328834, "avg_mpg_min": 31.491150545150028, "avg_mpg_max": 33.43499898677152}, "sampler_perf": {"mean_env_wait_ms": 1581.3850114561171, "mean_processing_ms": 429.98293130628656, "mean_inference_ms": 1.8573923165120951}, "off_policy_estimator": {}, "info": {"num_steps_trained": 231431424, "num_steps_sampled": 3230000, "sample_time_ms": 2044128.093, "load_time_ms": 453.819, "grad_time_ms": 185461.638, "update_time_ms": 8.061, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5858.8330078125, "policy_loss": 0.0008705683867447078, "vf_loss": 5858.8330078125, "vf_explained_var": 0.22340422868728638, "vf_preds": 119.17366790771484, "kl": 0.008127134293317795, "entropy": -0.591378927230835, "entropy_coeff": 0.0, "advantages": 1.1331854921081685e-06, "rewards": 1.9157532453536987}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 3230000, "episodes_total": 3230, "training_iteration": 190, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_22-44-28", "timestamp": 1599432268, "time_this_iter_s": 2234.527589082718, "time_total_s": 422870.53798890114, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc71ed90>", "on_episode_step": ".on_episode_step at 0x7f3afc71e6a8>", "on_episode_end": ".on_episode_end at 0x7f3af4f63158>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3af4f63a60>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 422870.53798890114, "timesteps_since_restore": 3230000, "iterations_since_restore": 190, "perf": {"cpu_util_percent": 89.10584354382658, "ram_util_percent": 30.77354696826893}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 145951.642291145, "episode_reward_min": 141309.66745409445, "episode_reward_mean": 143619.71588782957, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 625.2329250873341}, "policy_reward_mean": {"av": 454.91025272506255}, "custom_metrics": {"avg_speed_mean": 5.0228331942327245, "avg_speed_min": 4.917390803894327, "avg_speed_max": 5.167655353508848, "avg_speed_avs_mean": 5.017752625722605, "avg_speed_avs_min": 4.9035076125841295, "avg_speed_avs_max": 5.172621735862994, "avg_headway_avs_mean": 10.30007498233711, "avg_headway_avs_min": 9.136836664049085, "avg_headway_avs_max": 11.173276968431736, "std_headway_avs_mean": 8.282595401961412, "std_headway_avs_min": 7.775815310389579, "std_headway_avs_max": 9.13305438540454, "avg_accel_avs_mean": 0.19381605295786283, "avg_accel_avs_min": 0.18464743552315394, "avg_accel_avs_max": 0.20208162731210808, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 32.89057069432149, "avg_mpg_per_veh_min": 32.048273748273054, "avg_mpg_per_veh_max": 33.67751502333311, "num_cars_mean": 932.4789, "num_cars_min": 921.104, "num_cars_max": 940.861, "avg_mpg_mean": 32.7028913827886, "avg_mpg_min": 31.85612019005233, "avg_mpg_max": 33.43499898677152}, "sampler_perf": {"mean_env_wait_ms": 1581.2989675177532, "mean_processing_ms": 430.0854847918874, "mean_inference_ms": 1.8571627035810874}, "off_policy_estimator": {}, "info": {"num_steps_trained": 232659840, "num_steps_sampled": 3247000, "sample_time_ms": 2044885.366, "load_time_ms": 454.743, "grad_time_ms": 185868.416, "update_time_ms": 7.981, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5721.8759765625, "policy_loss": 0.00079538015415892, "vf_loss": 5721.87548828125, "vf_explained_var": 0.229010671377182, "vf_preds": 119.10775756835938, "kl": 0.007275908719748259, "entropy": -0.6080492734909058, "entropy_coeff": 0.0, "advantages": -1.1241882020840421e-05, "rewards": 1.896883487701416}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 3247000, "episodes_total": 3247, "training_iteration": 191, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_23-21-42", "timestamp": 1599434502, "time_this_iter_s": 2234.240368127823, "time_total_s": 425104.77835702896, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc704b70>", "on_episode_step": ".on_episode_step at 0x7f3afc704bf8>", "on_episode_end": ".on_episode_end at 0x7f3afc704e18>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc704f28>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 425104.77835702896, "timesteps_since_restore": 3247000, "iterations_since_restore": 191, "perf": {"cpu_util_percent": 88.61699654414076, "ram_util_percent": 30.798177819666982}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 145951.642291145, "episode_reward_min": 139839.38936092248, "episode_reward_mean": 143213.69656862377, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 625.2329250873341}, "policy_reward_mean": {"av": 454.4446803599155}, "custom_metrics": {"avg_speed_mean": 5.006613516461587, "avg_speed_min": 4.8710715706021634, "avg_speed_max": 5.167655353508848, "avg_speed_avs_mean": 5.0005211688208, "avg_speed_avs_min": 4.856976789743003, "avg_speed_avs_max": 5.172621735862994, "avg_headway_avs_mean": 10.345512823884576, "avg_headway_avs_min": 9.136836664049085, "avg_headway_avs_max": 11.173276968431736, "std_headway_avs_mean": 8.325999049015374, "std_headway_avs_min": 7.79241489161567, "std_headway_avs_max": 9.13305438540454, "avg_accel_avs_mean": 0.19387364205202637, "avg_accel_avs_min": 0.18464743552315394, "avg_accel_avs_max": 0.20208162731210808, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 32.78152296586332, "avg_mpg_per_veh_min": 31.55228508846777, "avg_mpg_per_veh_max": 33.67751502333311, "num_cars_mean": 933.2694899999999, "num_cars_min": 921.104, "num_cars_max": 945.353, "avg_mpg_mean": 32.594227707076364, "avg_mpg_min": 31.369696844349914, "avg_mpg_max": 33.43499898677152}, "sampler_perf": {"mean_env_wait_ms": 1581.1966404581724, "mean_processing_ms": 430.18708569518805, "mean_inference_ms": 1.8569083538561468}, "off_policy_estimator": {}, "info": {"num_steps_trained": 233896576, "num_steps_sampled": 3264000, "sample_time_ms": 2040640.327, "load_time_ms": 459.056, "grad_time_ms": 184391.107, "update_time_ms": 7.952, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5816.73779296875, "policy_loss": 0.0012683133827522397, "vf_loss": 5816.736328125, "vf_explained_var": 0.20583152770996094, "vf_preds": 117.87523651123047, "kl": 0.016243422403931618, "entropy": -0.6325480937957764, "entropy_coeff": 0.0, "advantages": 4.745330528521663e-08, "rewards": 1.8662147521972656}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 3264000, "episodes_total": 3264, "training_iteration": 192, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-06_23-58-28", "timestamp": 1599436708, "time_this_iter_s": 2205.4813945293427, "time_total_s": 427310.2597515583, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc672bf8>", "on_episode_step": ".on_episode_step at 0x7f3afc672e18>", "on_episode_end": ".on_episode_end at 0x7f3afc672268>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc672a60>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 427310.2597515583, "timesteps_since_restore": 3264000, "iterations_since_restore": 192, "perf": {"cpu_util_percent": 86.97344794651384, "ram_util_percent": 30.85781598217128}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 145951.642291145, "episode_reward_min": 139839.38936092248, "episode_reward_mean": 142864.08168564213, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 628.8156014335849}, "policy_reward_mean": {"av": 453.9691187977188}, "custom_metrics": {"avg_speed_mean": 4.991884207218752, "avg_speed_min": 4.8710715706021634, "avg_speed_max": 5.167655353508848, "avg_speed_avs_mean": 4.984852919095137, "avg_speed_avs_min": 4.856976789743003, "avg_speed_avs_max": 5.172621735862994, "avg_headway_avs_mean": 10.355566948958169, "avg_headway_avs_min": 9.136836664049085, "avg_headway_avs_max": 11.173276968431736, "std_headway_avs_mean": 8.327781418304852, "std_headway_avs_min": 7.79241489161567, "std_headway_avs_max": 9.13305438540454, "avg_accel_avs_mean": 0.19449104634848322, "avg_accel_avs_min": 0.18578947249684308, "avg_accel_avs_max": 0.20208162731210808, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 32.672391233085726, "avg_mpg_per_veh_min": 31.55228508846777, "avg_mpg_per_veh_max": 33.67751502333311, "num_cars_mean": 934.12625, "num_cars_min": 921.104, "num_cars_max": 945.353, "avg_mpg_mean": 32.48773898420676, "avg_mpg_min": 31.369696844349914, "avg_mpg_max": 33.43499898677152}, "sampler_perf": {"mean_env_wait_ms": 1581.0926069120846, "mean_processing_ms": 430.28877593027954, "mean_inference_ms": 1.856648985860469}, "off_policy_estimator": {}, "info": {"num_steps_trained": 235129088, "num_steps_sampled": 3281000, "sample_time_ms": 2035501.854, "load_time_ms": 460.955, "grad_time_ms": 184362.221, "update_time_ms": 7.982, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5887.09423828125, "policy_loss": 0.0008717796299606562, "vf_loss": 5887.09326171875, "vf_explained_var": 0.20244596898555756, "vf_preds": 117.01648712158203, "kl": 0.00714368699118495, "entropy": -0.6061747074127197, "entropy_coeff": 0.0, "advantages": 2.8639817628572928e-06, "rewards": 1.895279884338379}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 3281000, "episodes_total": 3281, "training_iteration": 193, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-07_00-35-20", "timestamp": 1599438920, "time_this_iter_s": 2211.8513176441193, "time_total_s": 429522.1110692024, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afff9b620>", "on_episode_step": ".on_episode_step at 0x7f3afff9b950>", "on_episode_end": ".on_episode_end at 0x7f3afff9bd08>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afff9bd90>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 429522.1110692024, "timesteps_since_restore": 3281000, "iterations_since_restore": 193, "perf": {"cpu_util_percent": 86.8898444938115, "ram_util_percent": 30.766423357664234}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 144892.4824601254, "episode_reward_min": 139839.38936092248, "episode_reward_mean": 142539.1240174186, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 628.8156014335849}, "policy_reward_mean": {"av": 453.62842599904087}, "custom_metrics": {"avg_speed_mean": 4.978542752798058, "avg_speed_min": 4.8710715706021634, "avg_speed_max": 5.112655527313065, "avg_speed_avs_mean": 4.970542542372942, "avg_speed_avs_min": 4.856976789743003, "avg_speed_avs_max": 5.111920468558395, "avg_headway_avs_mean": 10.413556520625713, "avg_headway_avs_min": 9.723755759650553, "avg_headway_avs_max": 11.173276968431736, "std_headway_avs_mean": 8.351111402000146, "std_headway_avs_min": 7.79241489161567, "std_headway_avs_max": 9.13305438540454, "avg_accel_avs_mean": 0.19447297644777314, "avg_accel_avs_min": 0.18578947249684308, "avg_accel_avs_max": 0.20208162731210808, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 32.59851731666495, "avg_mpg_per_veh_min": 31.55228508846777, "avg_mpg_per_veh_max": 33.390230467750996, "num_cars_mean": 934.5670599999999, "num_cars_min": 926.364, "num_cars_max": 945.353, "avg_mpg_mean": 32.41850210087909, "avg_mpg_min": 31.369696844349914, "avg_mpg_max": 33.18031238764583}, "sampler_perf": {"mean_env_wait_ms": 1581.0081254993968, "mean_processing_ms": 430.39130057469964, "mean_inference_ms": 1.856404988096168}, "off_policy_estimator": {}, "info": {"num_steps_trained": 236354176, "num_steps_sampled": 3298000, "sample_time_ms": 2035527.329, "load_time_ms": 459.414, "grad_time_ms": 184085.666, "update_time_ms": 7.958, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5992.7421875, "policy_loss": 0.0007654099608771503, "vf_loss": 5992.7421875, "vf_explained_var": 0.20301635563373566, "vf_preds": 119.27242279052734, "kl": 0.007280403282493353, "entropy": -0.6043195724487305, "entropy_coeff": 0.0, "advantages": -6.290373221418122e-06, "rewards": 1.9231585264205933}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 3298000, "episodes_total": 3298, "training_iteration": 194, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-07_01-12-36", "timestamp": 1599441156, "time_this_iter_s": 2236.567283153534, "time_total_s": 431758.67835235596, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc570268>", "on_episode_step": ".on_episode_step at 0x7f3afc570158>", "on_episode_end": ".on_episode_end at 0x7f3afc5700d0>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3b003b3840>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 431758.67835235596, "timesteps_since_restore": 3298000, "iterations_since_restore": 194, "perf": {"cpu_util_percent": 88.89199372056515, "ram_util_percent": 30.7879748822606}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 143972.0659120221, "episode_reward_min": 139839.38936092248, "episode_reward_mean": 142216.93692483203, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 628.864782378412}, "policy_reward_mean": {"av": 453.1799659831497}, "custom_metrics": {"avg_speed_mean": 4.962008565368798, "avg_speed_min": 4.8710715706021634, "avg_speed_max": 5.049497839428162, "avg_speed_avs_mean": 4.953173677451148, "avg_speed_avs_min": 4.856976789743003, "avg_speed_avs_max": 5.046637503844102, "avg_headway_avs_mean": 10.399480129093265, "avg_headway_avs_min": 9.72344157196296, "avg_headway_avs_max": 11.173276968431736, "std_headway_avs_mean": 8.350024523456819, "std_headway_avs_min": 7.867925910076435, "std_headway_avs_max": 9.13305438540454, "avg_accel_avs_mean": 0.19456621333720517, "avg_accel_avs_min": 0.18732897783411898, "avg_accel_avs_max": 0.20208162731210808, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 32.501521954131945, "avg_mpg_per_veh_min": 31.55228508846777, "avg_mpg_per_veh_max": 33.258420009742835, "num_cars_mean": 935.69237, "num_cars_min": 927.12, "num_cars_max": 945.353, "avg_mpg_mean": 32.32632306037317, "avg_mpg_min": 31.369696844349914, "avg_mpg_max": 33.04283093828529}, "sampler_perf": {"mean_env_wait_ms": 1580.6638512013703, "mean_processing_ms": 430.37460548040394, "mean_inference_ms": 1.85587226828018}, "off_policy_estimator": {}, "info": {"num_steps_trained": 237587072, "num_steps_sampled": 3315000, "sample_time_ms": 1995159.773, "load_time_ms": 455.977, "grad_time_ms": 188691.789, "update_time_ms": 8.046, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5847.2177734375, "policy_loss": 0.0008716450538486242, "vf_loss": 5847.2177734375, "vf_explained_var": 0.19654138386249542, "vf_preds": 118.26114654541016, "kl": 0.007283129263669252, "entropy": -0.6283288598060608, "entropy_coeff": 0.0, "advantages": 1.3952450217402657e-06, "rewards": 1.8976686000823975}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 3315000, "episodes_total": 3315, "training_iteration": 195, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-07_01-43-40", "timestamp": 1599443020, "time_this_iter_s": 1863.5672640800476, "time_total_s": 433622.245616436, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc6c0e18>", "on_episode_step": ".on_episode_step at 0x7f3afc6c0d90>", "on_episode_end": ".on_episode_end at 0x7f3afc6c0b70>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc6c0ae8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 433622.245616436, "timesteps_since_restore": 3315000, "iterations_since_restore": 195, "perf": {"cpu_util_percent": 50.06723163841808, "ram_util_percent": 22.629981167608282}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 143972.0659120221, "episode_reward_min": 139839.38936092248, "episode_reward_mean": 142157.8952087122, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 628.864782378412}, "policy_reward_mean": {"av": 453.0495736143546}, "custom_metrics": {"avg_speed_mean": 4.955591488653835, "avg_speed_min": 4.8710715706021634, "avg_speed_max": 5.049497839428162, "avg_speed_avs_mean": 4.946583705904859, "avg_speed_avs_min": 4.856976789743003, "avg_speed_avs_max": 5.046637503844102, "avg_headway_avs_mean": 10.3552061585006, "avg_headway_avs_min": 9.72344157196296, "avg_headway_avs_max": 11.173276968431736, "std_headway_avs_mean": 8.335070059242568, "std_headway_avs_min": 7.867925910076435, "std_headway_avs_max": 9.13305438540454, "avg_accel_avs_mean": 0.19460436932547576, "avg_accel_avs_min": 0.18732897783411898, "avg_accel_avs_max": 0.20208162731210808, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 32.48023768216949, "avg_mpg_per_veh_min": 31.55228508846777, "avg_mpg_per_veh_max": 33.258420009742835, "num_cars_mean": 936.25685, "num_cars_min": 927.12, "num_cars_max": 945.353, "avg_mpg_mean": 32.30936491090834, "avg_mpg_min": 31.369696844349914, "avg_mpg_max": 33.04283093828529}, "sampler_perf": {"mean_env_wait_ms": 1579.935854548022, "mean_processing_ms": 430.2340252435274, "mean_inference_ms": 1.854903507439824}, "off_policy_estimator": {}, "info": {"num_steps_trained": 238818816, "num_steps_sampled": 3332000, "sample_time_ms": 1938086.073, "load_time_ms": 453.247, "grad_time_ms": 193341.429, "update_time_ms": 8.099, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5800.26953125, "policy_loss": 0.0008537829271517694, "vf_loss": 5800.26904296875, "vf_explained_var": 0.19886307418346405, "vf_preds": 118.20645904541016, "kl": 0.00690584909170866, "entropy": -0.6313248872756958, "entropy_coeff": 0.0, "advantages": -3.409001010368229e-06, "rewards": 1.8989100456237793}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 3332000, "episodes_total": 3332, "training_iteration": 196, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-07_02-12-12", "timestamp": 1599444732, "time_this_iter_s": 1711.737869977951, "time_total_s": 435333.98348641396, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc627c80>", "on_episode_step": ".on_episode_step at 0x7f3afc627048>", "on_episode_end": ".on_episode_end at 0x7f3afc627840>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc6276a8>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 435333.98348641396, "timesteps_since_restore": 3332000, "iterations_since_restore": 196, "perf": {"cpu_util_percent": 42.13314191960624, "ram_util_percent": 19.228999179655457}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 143972.0659120221, "episode_reward_min": 139839.38936092248, "episode_reward_mean": 142067.16049870205, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 628.864782378412}, "policy_reward_mean": {"av": 453.02028220249395}, "custom_metrics": {"avg_speed_mean": 4.946356471120218, "avg_speed_min": 4.857451056897386, "avg_speed_max": 5.046846518760918, "avg_speed_avs_mean": 4.937418406220171, "avg_speed_avs_min": 4.846951245590176, "avg_speed_avs_max": 5.04369663370076, "avg_headway_avs_mean": 10.30130491134108, "avg_headway_avs_min": 9.655152911677957, "avg_headway_avs_max": 10.995244306095028, "std_headway_avs_mean": 8.272216058534157, "std_headway_avs_min": 7.867925910076435, "std_headway_avs_max": 8.887915636350629, "avg_accel_avs_mean": 0.19428257770292348, "avg_accel_avs_min": 0.18732897783411898, "avg_accel_avs_max": 0.202062240314735, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 32.44746848427413, "avg_mpg_per_veh_min": 31.49469880215364, "avg_mpg_per_veh_max": 33.258420009742835, "num_cars_mean": 937.096, "num_cars_min": 928.241, "num_cars_max": 945.353, "avg_mpg_mean": 32.27816749087624, "avg_mpg_min": 31.331697379132887, "avg_mpg_max": 33.04283093828529}, "sampler_perf": {"mean_env_wait_ms": 1578.8321339714014, "mean_processing_ms": 429.97019782542156, "mean_inference_ms": 1.85350332303187}, "off_policy_estimator": {}, "info": {"num_steps_trained": 240057344, "num_steps_sampled": 3349000, "sample_time_ms": 1886518.641, "load_time_ms": 452.549, "grad_time_ms": 198008.102, "update_time_ms": 8.465, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5621.8427734375, "policy_loss": 0.0008274067076854408, "vf_loss": 5621.84228515625, "vf_explained_var": 0.19217316806316376, "vf_preds": 116.21123504638672, "kl": 0.007714997511357069, "entropy": -0.6390237808227539, "entropy_coeff": 0.0, "advantages": -3.9182368709589355e-06, "rewards": 1.8716481924057007}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 3349000, "episodes_total": 3349, "training_iteration": 197, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-07_02-41-12", "timestamp": 1599446472, "time_this_iter_s": 1739.7935011386871, "time_total_s": 437073.77698755264, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc6c02f0>", "on_episode_step": ".on_episode_step at 0x7f3afc6c0400>", "on_episode_end": ".on_episode_end at 0x7f3afc6c0c80>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc6c0268>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 437073.77698755264, "timesteps_since_restore": 3349000, "iterations_since_restore": 197, "perf": {"cpu_util_percent": 41.54546187979024, "ram_util_percent": 19.144009681323116}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 143972.0659120221, "episode_reward_min": 139717.41115387742, "episode_reward_mean": 141972.0699042781, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 628.864782378412}, "policy_reward_mean": {"av": 452.8759128019334}, "custom_metrics": {"avg_speed_mean": 4.939301202890724, "avg_speed_min": 4.810730461009447, "avg_speed_max": 5.046846518760918, "avg_speed_avs_mean": 4.930058009273967, "avg_speed_avs_min": 4.796229996766206, "avg_speed_avs_max": 5.04369663370076, "avg_headway_avs_mean": 10.248489405841887, "avg_headway_avs_min": 9.434953381742561, "avg_headway_avs_max": 10.995244306095028, "std_headway_avs_mean": 8.218349226038722, "std_headway_avs_min": 7.867925910076435, "std_headway_avs_max": 8.626775396066352, "avg_accel_avs_mean": 0.19379413363367484, "avg_accel_avs_min": 0.18638883336989293, "avg_accel_avs_max": 0.20183312345044332, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 32.39820980722774, "avg_mpg_per_veh_min": 31.446857877676887, "avg_mpg_per_veh_max": 33.258420009742835, "num_cars_mean": 937.7155999999999, "num_cars_min": 928.241, "num_cars_max": 949.963, "avg_mpg_mean": 32.23004843406281, "avg_mpg_min": 31.288200571666945, "avg_mpg_max": 33.04283093828529}, "sampler_perf": {"mean_env_wait_ms": 1577.3804836470983, "mean_processing_ms": 429.5850479115072, "mean_inference_ms": 1.851699386472906}, "off_policy_estimator": {}, "info": {"num_steps_trained": 241300992, "num_steps_sampled": 3366000, "sample_time_ms": 1836659.896, "load_time_ms": 455.866, "grad_time_ms": 203360.686, "update_time_ms": 8.395, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5631.9970703125, "policy_loss": 0.000882848515175283, "vf_loss": 5631.99609375, "vf_explained_var": 0.17272737622261047, "vf_preds": 115.14289855957031, "kl": 0.008189043030142784, "entropy": -0.6900065541267395, "entropy_coeff": 0.0, "advantages": 1.1319790473862668e-06, "rewards": 1.8506888151168823}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 3366000, "episodes_total": 3366, "training_iteration": 198, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-07_03-10-44", "timestamp": 1599448244, "time_this_iter_s": 1771.7869844436646, "time_total_s": 438845.5639719963, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5f9730>", "on_episode_step": ".on_episode_step at 0x7f3afc5f9048>", "on_episode_end": ".on_episode_end at 0x7f3afc5f9268>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5f9598>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 438845.5639719963, "timesteps_since_restore": 3366000, "iterations_since_restore": 198, "perf": {"cpu_util_percent": 41.05570522979398, "ram_util_percent": 19.118264659271}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 143972.0659120221, "episode_reward_min": 139451.7604388565, "episode_reward_mean": 141570.86282297314, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 628.864782378412}, "policy_reward_mean": {"av": 452.3608858096023}, "custom_metrics": {"avg_speed_mean": 4.920968979860779, "avg_speed_min": 4.810730461009447, "avg_speed_max": 5.046846518760918, "avg_speed_avs_mean": 4.910683057441147, "avg_speed_avs_min": 4.794536006229754, "avg_speed_avs_max": 5.04369663370076, "avg_headway_avs_mean": 10.230648098614306, "avg_headway_avs_min": 9.434953381742561, "avg_headway_avs_max": 10.995244306095028, "std_headway_avs_mean": 8.199080698282437, "std_headway_avs_min": 7.776007110921273, "std_headway_avs_max": 8.594331439695315, "avg_accel_avs_mean": 0.19436329100592156, "avg_accel_avs_min": 0.18638883336989293, "avg_accel_avs_max": 0.20383163562180903, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 32.25649526739194, "avg_mpg_per_veh_min": 31.438205332080592, "avg_mpg_per_veh_max": 33.13264746109683, "num_cars_mean": 938.88325, "num_cars_min": 928.241, "num_cars_max": 949.963, "avg_mpg_mean": 32.092652477623844, "avg_mpg_min": 31.28034963776727, "avg_mpg_max": 32.99184943844368}, "sampler_perf": {"mean_env_wait_ms": 1575.5821500532254, "mean_processing_ms": 429.0796388797781, "mean_inference_ms": 1.8494923058679589}, "off_policy_estimator": {}, "info": {"num_steps_trained": 242545408, "num_steps_sampled": 3383000, "sample_time_ms": 1786139.978, "load_time_ms": 465.407, "grad_time_ms": 208193.09, "update_time_ms": 8.301, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5235.1416015625, "policy_loss": 0.0009047583444043994, "vf_loss": 5235.14111328125, "vf_explained_var": 0.17976577579975128, "vf_preds": 116.92879486083984, "kl": 0.007198583334684372, "entropy": -0.6606705784797668, "entropy_coeff": 0.0, "advantages": -3.958111392421415e-06, "rewards": 1.8477400541305542}}}, "timesteps_this_iter": 17000, "done": false, "timesteps_total": 3383000, "episodes_total": 3383, "training_iteration": 199, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-07_03-39-46", "timestamp": 1599449986, "time_this_iter_s": 1742.589968919754, "time_total_s": 440588.15394091606, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5f96a8>", "on_episode_step": ".on_episode_step at 0x7f3afc5f9ea0>", "on_episode_end": ".on_episode_end at 0x7f3afc5f98c8>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5f9950>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 440588.15394091606, "timesteps_since_restore": 3383000, "iterations_since_restore": 199, "perf": {"cpu_util_percent": 41.631494160289975, "ram_util_percent": 19.244301248489734}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} +{"episode_reward_max": 143271.76963134107, "episode_reward_min": 139199.53357968212, "episode_reward_mean": 140960.2703847606, "episode_len_mean": 1000.0, "episodes_this_iter": 17, "policy_reward_min": {"av": 0.0}, "policy_reward_max": {"av": 628.864782378412}, "policy_reward_mean": {"av": 451.4918496677257}, "custom_metrics": {"avg_speed_mean": 4.890552623264747, "avg_speed_min": 4.808053113741696, "avg_speed_max": 5.004589976699192, "avg_speed_avs_mean": 4.878829837408377, "avg_speed_avs_min": 4.7915005780914415, "avg_speed_avs_max": 4.998471628440937, "avg_headway_avs_mean": 10.196383673075395, "avg_headway_avs_min": 9.434953381742561, "avg_headway_avs_max": 10.995244306095028, "std_headway_avs_mean": 8.194724849986383, "std_headway_avs_min": 7.776007110921273, "std_headway_avs_max": 8.594331439695315, "avg_accel_avs_mean": 0.19483175288859475, "avg_accel_avs_min": 0.18638883336989293, "avg_accel_avs_max": 0.20383163562180903, "avg_energy_per_veh_mean": NaN, "avg_energy_per_veh_min": NaN, "avg_energy_per_veh_max": NaN, "avg_mpg_per_veh_mean": 32.071155427991286, "avg_mpg_per_veh_min": 31.2876363783236, "avg_mpg_per_veh_max": 32.90594170165136, "num_cars_mean": 940.9095600000002, "num_cars_min": 930.716, "num_cars_max": 949.963, "avg_mpg_mean": 31.91280932316943, "avg_mpg_min": 31.13019292451482, "avg_mpg_max": 32.70980552063074}, "sampler_perf": {"mean_env_wait_ms": 1573.4610962736751, "mean_processing_ms": 428.46960512649775, "mean_inference_ms": 1.8469027545382468}, "off_policy_estimator": {}, "info": {"num_steps_trained": 243790720, "num_steps_sampled": 3400000, "sample_time_ms": 1729949.592, "load_time_ms": 468.768, "grad_time_ms": 213642.64, "update_time_ms": 8.78, "learner": {"av": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 0.0005000000237487257, "total_loss": 5308.396484375, "policy_loss": 0.0008936258964240551, "vf_loss": 5308.39599609375, "vf_explained_var": 0.17144528031349182, "vf_preds": 117.72993469238281, "kl": 0.007077354472130537, "entropy": -0.6869407892227173, "entropy_coeff": 0.0, "advantages": 1.0725333595473785e-05, "rewards": 1.8426936864852905}}}, "timesteps_this_iter": 17000, "done": true, "timesteps_total": 3400000, "episodes_total": 3400, "training_iteration": 200, "experiment_id": "00b547e0d51d48debde5c46f678a10c8", "date": "2020-09-07_04-08-33", "timestamp": 1599451713, "time_this_iter_s": 1727.1710240840912, "time_total_s": 442315.32496500015, "pid": 22566, "hostname": "ip-172-31-29-2", "node_ip": "172.31.29.2", "config": {"num_workers": 17, "num_envs_per_worker": 1, "sample_batch_size": 200, "batch_mode": "truncate_episodes", "num_gpus": 0, "train_batch_size": 17000, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [32, 32], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_action_dist": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.995, "horizon": 1000, "soft_horizon": false, "no_done_at_end": true, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.0025,\n \"accumulated_reward\": true,\n \"accumulated_reward_bonus\": 20,\n \"accumulated_reward_interval_dist\": 50,\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"late_penalty\": false,\n \"late_penalty_add_state\": true,\n \"late_penalty_steps\": 500,\n \"late_penalty_value\": -1.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 8,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_downstream_speed\": 5.0,\n \"max_num_agents\": 10,\n \"min_downstream_speed\": 5.0,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"no_control_edges\": [\n \"ghost0\",\n \"119257908#3\"\n ],\n \"penalize_accel\": false,\n \"penalize_headway\": false,\n \"penalize_headway_threshold\": 150,\n \"penalize_headway_value\": -1.0,\n \"penalize_stops\": false,\n \"randomize_downstream_speed\": true,\n \"reroute_on_exit\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 5.0,\n \"stop_penalty\": 0.01,\n \"target_velocity\": 5.0,\n \"warm_down\": false,\n \"warm_down_steps\": 500\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 600\n },\n \"env_name\": \"flow.envs.multiagent.i210.I210MultiEnv\",\n \"exp_tag\": \"I_210_subnetwork\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": [\n \"ghost0\",\n \"119257914\",\n \"119257908#0\",\n \"119257908#1-AddedOnRampEdge\",\n \"119257908#1\",\n \"119257908#1-AddedOffRampEdge\",\n \"119257908#2\",\n \"119257908#3\",\n \"27414345\",\n \"27414342#0\",\n \"27414342#1-AddedOnRampEdge\",\n \"173381935\"\n ],\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"ghost_edge\": true,\n \"on_ramp\": false\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_0\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 0,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_1\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_2\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 1,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_3\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_4\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 2,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_5\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_6\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 3,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_7\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_8\",\n \"vehsPerHour\": 1845,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": 4,\n \"departSpeed\": 25.5,\n \"edge\": \"ghost0\",\n \"end\": 86400,\n \"name\": \"flow_9\",\n \"vehsPerHour\": 205,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": \"/home/ubuntu/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml\"\n },\n \"network\": \"flow.networks.i210_subnetwork.I210SubNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": true,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": true,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ],\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 1621\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {\n \"display_warnings\": false,\n \"fail_safe\": [\n \"obey_speed_limit\",\n \"safe_velocity\",\n \"feasible_accel\"\n ]\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 12\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": ""}, "env": "I210MultiEnv-v2", "clip_rewards": null, "clip_actions": true, "preprocessor_pref": "deepmind", "lr": 0.0005, "monitor": false, "log_level": "WARN", "callbacks": {"on_episode_start": ".on_episode_start at 0x7f3afc5a4268>", "on_episode_step": ".on_episode_step at 0x7f3afc5a4950>", "on_episode_end": ".on_episode_end at 0x7f3afc5a4b70>", "on_sample_end": null, "on_train_result": ".on_train_result at 0x7f3afc5a4f28>", "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "eager": false, "eager_tracing": false, "no_eager_on_workers": false, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "memory": 0, "object_store_memory": 0, "memory_per_worker": 0, "object_store_memory_per_worker": 0, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {"av": ["", "Box(5,)", "Box(1,)", {}]}, "policy_mapping_fn": "", "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 442315.32496500015, "timesteps_since_restore": 3400000, "iterations_since_restore": 200, "perf": {"cpu_util_percent": 42.11548780487805, "ram_util_percent": 19.16239837398374}, "num_healthy_workers": 17, "trial_id": "e504e8cc", "experiment_tag": "0_lr=0.0005"} diff --git a/examples/exp_configs/non_rl/highway_single.py b/examples/exp_configs/non_rl/highway_single.py index 0ced89f27..7e7ec5a2e 100644 --- a/examples/exp_configs/non_rl/highway_single.py +++ b/examples/exp_configs/non_rl/highway_single.py @@ -1,15 +1,20 @@ -"""Example of an open network with human-driven vehicles.""" +"""Example of an open network with human-driven vehicles and a wave.""" + +import numpy as np + +from examples.exp_configs.rl.multiagent.multiagent_straight_road import additional_env_params from flow.controllers import IDMController +from flow.controllers.velocity_controllers import FollowerStopper from flow.core.params import EnvParams from flow.core.params import NetParams from flow.core.params import InitialConfig from flow.core.params import InFlows from flow.core.params import VehicleParams from flow.core.params import SumoParams -from flow.core.params import SumoLaneChangeParams +from flow.core.rewards import instantaneous_mpg from flow.core.params import SumoCarFollowingParams from flow.networks import HighwayNetwork -from flow.envs import TestEnv +from flow.envs.multiagent import StraightRoadTestEnv from flow.networks.highway import ADDITIONAL_NET_PARAMS # the speed of vehicles entering the network @@ -22,6 +27,8 @@ HORIZON = 1500 # whether to include noise in the car-following models INCLUDE_NOISE = True +# fraction of vehicles that are follower-stoppers. 0.10 corresponds to 10% +PENETRATION_RATE = 0.0 additional_net_params = ADDITIONAL_NET_PARAMS.copy() additional_net_params.update({ @@ -48,32 +55,60 @@ acceleration_controller=(IDMController, { 'a': 1.3, 'b': 2.0, - 'noise': 0.3 if INCLUDE_NOISE else 0.0 + 'noise': 0.3 if INCLUDE_NOISE else 0.0, + "display_warnings": False, + "fail_safe": ['obey_speed_limit', 'safe_velocity', 'feasible_accel'], }), car_following_params=SumoCarFollowingParams( - min_gap=0.5 - ), - lane_change_params=SumoLaneChangeParams( - model="SL2015", - lc_sublane=2.0, + min_gap=0.5, + speed_mode=12 # right of way at intersections + obey limits on deceleration ), ) +if PENETRATION_RATE > 0.0: + vehicles.add( + "av", + color='red', + car_following_params=SumoCarFollowingParams( + min_gap=0.5, + speed_mode=12 # right of way at intersections + obey limits on deceleration + ), + num_vehicles=0, + acceleration_controller=(FollowerStopper, { + "v_des": 5.0, + "control_length": [500, 2300], + "display_warnings": False, + "fail_safe": ['obey_speed_limit', 'safe_velocity', 'feasible_accel'], + }), + ) + inflows = InFlows() + inflows.add( veh_type="human", edge="highway_0", - vehs_per_hour=TRAFFIC_FLOW, + vehs_per_hour=int(TRAFFIC_FLOW * (1 - PENETRATION_RATE)), depart_lane="free", depart_speed=TRAFFIC_SPEED, name="idm_highway_inflow") +if PENETRATION_RATE > 0.0: + inflows.add( + veh_type="av", + edge="highway_0", + vehs_per_hour=int(TRAFFIC_FLOW * PENETRATION_RATE), + depart_lane="free", + depart_speed=TRAFFIC_SPEED, + name="av_highway_inflow") + +# SET UP FLOW PARAMETERS + flow_params = dict( # name of the experiment exp_tag='highway-single', # name of the flow environment the experiment is running on - env_name=TestEnv, + env_name=StraightRoadTestEnv, # name of the network class the experiment is running on network=HighwayNetwork, @@ -86,6 +121,7 @@ horizon=HORIZON, warmup_steps=500, sims_per_step=3, + additional_params=additional_env_params ), # sumo-related parameters (see flow.core.params.SumoParams) @@ -111,3 +147,13 @@ # reset (see flow.core.params.InitialConfig) initial=InitialConfig(), ) + +custom_callables = { + "avg_merge_speed": lambda env: np.nan_to_num(np.mean( + env.k.vehicle.get_speed(env.k.vehicle.get_ids()))), + "avg_outflow": lambda env: np.nan_to_num( + env.k.vehicle.get_outflow_rate(120)), + "miles_per_gallon": lambda env: np.nan_to_num( + instantaneous_mpg(env, env.k.vehicle.get_ids(), gain=1.0) + ) +} diff --git a/examples/exp_configs/non_rl/i210_subnetwork.py b/examples/exp_configs/non_rl/i210_subnetwork.py index b0c58c894..149169842 100644 --- a/examples/exp_configs/non_rl/i210_subnetwork.py +++ b/examples/exp_configs/non_rl/i210_subnetwork.py @@ -2,18 +2,24 @@ import os import numpy as np -from flow.controllers import IDMController -from flow.controllers import I210Router +from flow.controllers.lane_change_controllers import AILaneChangeController, SimLaneChangeController +from flow.controllers.car_following_models import IDMController +from flow.controllers.velocity_controllers import FollowerStopper +from flow.controllers.routing_controllers import I210Router from flow.core.params import SumoParams from flow.core.params import EnvParams from flow.core.params import NetParams from flow.core.params import SumoLaneChangeParams +from flow.core.params import SumoCarFollowingParams from flow.core.params import VehicleParams from flow.core.params import InitialConfig from flow.core.params import InFlows +from flow.core.rewards import instantaneous_mpg +from examples.exp_configs.rl.multiagent.multiagent_i210 import additional_env_params +from flow.envs.multiagent import I210TestEnv +from flow.networks import I210SubNetwork +from flow.networks.i210_subnetwork import EDGES_DISTRIBUTION import flow.config as config -from flow.envs import TestEnv -from flow.networks.i210_subnetwork import I210SubNetwork, EDGES_DISTRIBUTION # =========================================================================== # # Specify some configurable constants. # @@ -27,12 +33,22 @@ ON_RAMP = False # the inflow rate of vehicles (in veh/hr) INFLOW_RATE = 2050 +# if this is false, the AVs all come in as a wave, otherwise they are randomly scattered +ENTER_AS_LINE = False +# on-ramp inflow_rate +ON_RAMP_INFLOW_RATE = 500 # the speed of inflowing vehicles from the main edge (in m/s) INFLOW_SPEED = 25.5 +# fraction of vehicles that are follower-stoppers. 0.10 corresponds to 10% +PENETRATION_RATE = 0.0 +# desired speed of the follower stopper vehicles +V_DES = 5.0 # horizon over which to run the env HORIZON = 1500 # steps to run before follower-stopper is allowed to take control WARMUP_STEPS = 600 +# whether to use the Rutgers lane change model +RUTGERS_LC_MODEL = False # =========================================================================== # # Specify the path to the network template. # @@ -63,43 +79,104 @@ # =========================================================================== # vehicles = VehicleParams() + +if RUTGERS_LC_MODEL: + lc_controller = (AILaneChangeController, {}) +else: + lc_controller = (SimLaneChangeController, {}) + vehicles.add( "human", num_vehicles=0, lane_change_params=SumoLaneChangeParams( - lane_change_mode="strategic", + lane_change_mode="sumo_default", ), + # this is only right of way on + car_following_params=SumoCarFollowingParams( + min_gap=0.5, + speed_mode=12 # right of way at intersections + obey limits on deceleration + ), + lane_change_controller=lc_controller, acceleration_controller=(IDMController, { "a": 1.3, "b": 2.0, "noise": 0.3, + "display_warnings": False, + "fail_safe": ['obey_speed_limit', 'safe_velocity', 'feasible_accel'], + }), + routing_controller=(I210Router, {}) if ON_RAMP else None, +) + +vehicles.add( + "av", + num_vehicles=0, + color="red", + # this is only right of way on + car_following_params=SumoCarFollowingParams( + min_gap=0.5, + speed_mode=12 # right of way at intersections + obey limits on deceleration + ), + acceleration_controller=(FollowerStopper, { + "v_des": V_DES, + "no_control_edges": ["ghost0", "119257908#3"], + "fail_safe": ['obey_speed_limit', 'safe_velocity', 'feasible_accel'], }), routing_controller=(I210Router, {}) if ON_RAMP else None, ) inflow = InFlows() + # main highway -for lane in [0, 1, 2, 3, 4]: +highway_start_edge = "ghost0" if WANT_GHOST_CELL else "119257914" + +if ENTER_AS_LINE: + for lane in [0, 1, 2, 3, 4]: + inflow.add( + veh_type="human", + edge=highway_start_edge, + vehs_per_hour=INFLOW_RATE * (1 - PENETRATION_RATE), + depart_lane=lane, + depart_speed=INFLOW_SPEED) + + if PENETRATION_RATE > 0.0: + inflow.add( + veh_type="av", + edge=highway_start_edge, + vehs_per_hour=INFLOW_RATE * PENETRATION_RATE, + depart_lane=lane, + depart_speed=INFLOW_SPEED) +else: inflow.add( veh_type="human", - edge="ghost0" if WANT_GHOST_CELL else "119257914", - vehs_per_hour=INFLOW_RATE, - departLane=lane, - departSpeed=INFLOW_SPEED) + edge=highway_start_edge, + vehs_per_hour=INFLOW_RATE * 5 * (1 - PENETRATION_RATE), + depart_lane="best", + depart_speed=INFLOW_SPEED) + + if PENETRATION_RATE > 0.0: + inflow.add( + veh_type="av", + edge=highway_start_edge, + vehs_per_hour=INFLOW_RATE * 5 * PENETRATION_RATE, + depart_lane="best", + depart_speed=INFLOW_SPEED) + # on ramp if ON_RAMP: inflow.add( veh_type="human", edge="27414345", - vehs_per_hour=500, - departLane="random", - departSpeed=10) - inflow.add( - veh_type="human", - edge="27414342#0", - vehs_per_hour=500, - departLane="random", - departSpeed=10) + vehs_per_hour=int(ON_RAMP_INFLOW_RATE * (1 - PENETRATION_RATE)), + depart_speed=10, + ) + + if PENETRATION_RATE > 0.0: + inflow.add( + veh_type="av", + edge="27414345", + vehs_per_hour=int(ON_RAMP_INFLOW_RATE * PENETRATION_RATE), + depart_lane="random", + depart_speed=10) # =========================================================================== # # Generate the flow_params dict with all relevant simulation information. # @@ -110,7 +187,7 @@ exp_tag='I-210_subnetwork', # name of the flow environment the experiment is running on - env_name=TestEnv, + env_name=I210TestEnv, # name of the network class the experiment is running on network=I210SubNetwork, @@ -122,15 +199,17 @@ sim=SumoParams( sim_step=0.4, render=False, - color_by_speed=True, - use_ballistic=True + color_by_speed=False, + use_ballistic=True, + overtake_right=True, ), # environment related parameters (see flow.core.params.EnvParams) env=EnvParams( horizon=HORIZON, warmup_steps=WARMUP_STEPS, - sims_per_step=3 + sims_per_step=3, + additional_params=additional_env_params, ), # network-related parameters (see flow.core.params.NetParams and the @@ -160,14 +239,21 @@ # =========================================================================== # edge_id = "119257908#1-AddedOnRampEdge" + + +def valid_ids(env, veh_ids): + """Return the names of vehicles within the controllable edges.""" + return [ + veh_id for veh_id in veh_ids + if env.k.vehicle.get_edge(veh_id) not in ["ghost0", "119257908#3"] + ] + + custom_callables = { "avg_merge_speed": lambda env: np.nan_to_num(np.mean( - env.k.vehicle.get_speed(env.k.vehicle.get_ids_by_edge(edge_id)))), + env.k.vehicle.get_speed(valid_ids(env, env.k.vehicle.get_ids())))), "avg_outflow": lambda env: np.nan_to_num( env.k.vehicle.get_outflow_rate(120)), - # we multiply by 5 to account for the vehicle length and by 1000 to convert - # into veh/km - "avg_density": lambda env: 5 * 1000 * len(env.k.vehicle.get_ids_by_edge( - edge_id)) / (env.k.network.edge_length(edge_id) - * env.k.network.num_lanes(edge_id)), + "instantaneous_mpg": lambda env: instantaneous_mpg( + env, valid_ids(env, env.k.vehicle.get_ids()), gain=1.0), } diff --git a/examples/exp_configs/rl/multiagent/adversarial_figure_eight.py b/examples/exp_configs/rl/multiagent/adversarial_figure_eight.py index 4fb81ce97..a7834c6cd 100644 --- a/examples/exp_configs/rl/multiagent/adversarial_figure_eight.py +++ b/examples/exp_configs/rl/multiagent/adversarial_figure_eight.py @@ -8,7 +8,7 @@ # the negative of the AV reward from copy import deepcopy -from ray.rllib.agents.ppo.ppo_policy import PPOTFPolicy +from ray.rllib.agents.ppo.ppo_tf_policy import PPOTFPolicy from flow.controllers import ContinuousRouter from flow.controllers import IDMController from flow.controllers import RLController diff --git a/examples/exp_configs/rl/multiagent/lord_of_the_rings.py b/examples/exp_configs/rl/multiagent/lord_of_the_rings.py index e7688c87d..866d915ce 100644 --- a/examples/exp_configs/rl/multiagent/lord_of_the_rings.py +++ b/examples/exp_configs/rl/multiagent/lord_of_the_rings.py @@ -3,7 +3,7 @@ Creates a set of stabilizing the ring experiments to test if more agents -> fewer needed batches """ -from ray.rllib.agents.ppo.ppo_policy import PPOTFPolicy +from ray.rllib.agents.ppo.ppo_tf_policy import PPOTFPolicy from flow.controllers import ContinuousRouter from flow.controllers import IDMController from flow.controllers import RLController diff --git a/examples/exp_configs/rl/multiagent/multiagent_figure_eight.py b/examples/exp_configs/rl/multiagent/multiagent_figure_eight.py index 0579bb978..2b18bf61b 100644 --- a/examples/exp_configs/rl/multiagent/multiagent_figure_eight.py +++ b/examples/exp_configs/rl/multiagent/multiagent_figure_eight.py @@ -1,5 +1,5 @@ """Figure eight example.""" -from ray.rllib.agents.ppo.ppo_policy import PPOTFPolicy +from ray.rllib.agents.ppo.ppo_tf_policy import PPOTFPolicy from ray.tune.registry import register_env from flow.core.params import SumoParams, EnvParams, InitialConfig, NetParams diff --git a/examples/exp_configs/rl/multiagent/multiagent_highway.py b/examples/exp_configs/rl/multiagent/multiagent_highway.py index cec0b3fba..353eccb2a 100644 --- a/examples/exp_configs/rl/multiagent/multiagent_highway.py +++ b/examples/exp_configs/rl/multiagent/multiagent_highway.py @@ -3,7 +3,7 @@ Trains a non-constant number of agents, all sharing the same policy, on the highway with ramps network. """ -from ray.rllib.agents.ppo.ppo_policy import PPOTFPolicy +from ray.rllib.agents.ppo.ppo_tf_policy import PPOTFPolicy from flow.controllers import RLController from flow.core.params import EnvParams, NetParams, InitialConfig, InFlows, \ VehicleParams, SumoParams, \ diff --git a/examples/exp_configs/rl/multiagent/multiagent_i210.py b/examples/exp_configs/rl/multiagent/multiagent_i210.py index a6d194708..70d1cb8f0 100644 --- a/examples/exp_configs/rl/multiagent/multiagent_i210.py +++ b/examples/exp_configs/rl/multiagent/multiagent_i210.py @@ -4,118 +4,283 @@ highway with ramps network. """ import os +import numpy as np -from ray.rllib.agents.ppo.ppo_policy import PPOTFPolicy from ray.tune.registry import register_env +from flow.controllers import RLController +from flow.controllers.routing_controllers import I210Router +from flow.controllers.car_following_models import IDMController import flow.config as config -from flow.controllers.rlcontroller import RLController from flow.core.params import EnvParams from flow.core.params import NetParams from flow.core.params import InitialConfig from flow.core.params import InFlows from flow.core.params import VehicleParams from flow.core.params import SumoParams +from flow.core.params import SumoCarFollowingParams from flow.core.params import SumoLaneChangeParams -from flow.networks.i210_subnetwork import I210SubNetwork, EDGES_DISTRIBUTION +from flow.core.rewards import energy_consumption from flow.envs.multiagent.i210 import I210MultiEnv, ADDITIONAL_ENV_PARAMS from flow.utils.registry import make_create_env +from flow.networks.i210_subnetwork import I210SubNetwork, EDGES_DISTRIBUTION -# SET UP PARAMETERS FOR THE SIMULATION +# =========================================================================== # +# Specify some configurable constants. # +# =========================================================================== # -# number of training iterations -N_TRAINING_ITERATIONS = 200 -# number of rollouts per training iteration -N_ROLLOUTS = 2 -# number of steps per rollout -HORIZON = 500 -# number of parallel workers -N_CPUS = 1 +# whether to include the downstream slow-down edge in the network as well as a +# ghost cell at the upstream edge +WANT_BOUNDARY_CONDITIONS = True +# whether to include vehicles on the on-ramp +ON_RAMP = False +# the inflow rate of vehicles (in veh/hr) +INFLOW_RATE = 2050 +# if this is false, the AVs all come in as a wave, otherwise they are randomly scattered +ENTER_AS_LINE = False +# the inflow rate on the on-ramp (in veh/hr) +ON_RAMP_INFLOW_RATE = 500 +# the speed of inflowing vehicles from the main edge (in m/s) +INFLOW_SPEED = 25.5 +# fraction of vehicles that are RL vehicles. 0.10 corresponds to 10% +PENETRATION_RATE = 0.05 +# desired speed of the vehicles in the network +V_DES = 5.0 +# horizon over which to run the env +HORIZON = 1000 +# steps to run before follower-stopper is allowed to take control +WARMUP_STEPS = 600 +# whether to turn off the fail safes for the human-driven vehicles +ALLOW_COLLISIONS = False -# percentage of autonomous vehicles compared to human vehicles on highway -PENETRATION_RATE = 10 +# =========================================================================== # +# Specify the path to the network template. # +# =========================================================================== # -# TODO: temporary fix +if WANT_BOUNDARY_CONDITIONS: + NET_TEMPLATE = os.path.join( + config.PROJECT_PATH, + "examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_" + "downstream.xml") +else: + NET_TEMPLATE = os.path.join( + config.PROJECT_PATH, + "examples/exp_configs/templates/sumo/test2.net.xml") edges_distribution = EDGES_DISTRIBUTION.copy() -edges_distribution.remove("ghost0") -# SET UP PARAMETERS FOR THE ENVIRONMENT +# =========================================================================== # +# Set up parameters for the environment. # +# =========================================================================== # + additional_env_params = ADDITIONAL_ENV_PARAMS.copy() additional_env_params.update({ - 'max_accel': 1, - 'max_decel': 1, - # configure the observation space. Look at the I210MultiEnv class for more info. + 'max_accel': 2.6, + 'max_decel': 4.5, + + # configure the observation space. Look at the I210MultiEnv class for more + # info. 'lead_obs': True, + # whether to add in a reward for the speed of nearby vehicles + "local_reward": True, + # whether to use the MPG reward. Otherwise, defaults to a target velocity + # reward + "mpg_reward": False, + # whether to use the MPJ reward. Otherwise, defaults to a target velocity + # reward + "mpj_reward": False, + # how many vehicles to look back for any reward + "look_back_length": 3, + # whether to reroute vehicles once they have exited + "reroute_on_exit": False, + 'target_velocity': 5.0, + # how many AVs there can be at once (this is only for centralized critics) + "max_num_agents": 10, + # which edges we shouldn't apply control on + "no_control_edges": ["ghost0", "119257908#3"], + + # whether to add a slight reward for opening up a gap that will be annealed + # out N iterations in + "headway_curriculum": False, + # how many timesteps to anneal the headway curriculum over + "headway_curriculum_iters": 100, + # weight of the headway reward + "headway_reward_gain": 2.0, + # desired time headway + "min_time_headway": 2.0, + + # whether to add a slight reward for traveling at a desired speed + "speed_curriculum": True, + # how many timesteps to anneal the headway curriculum over + "speed_curriculum_iters": 20, + # weight of the headway reward + "speed_reward_gain": 5.0, + # penalize stopped vehicles + "penalize_stops": False, + "stop_penalty": 0.01, + + # penalize accels + "penalize_accel": False, + "accel_penalty": (1 / 400.0), + + # control the range of speeds of the downstream edge + # if true, we vary the downstream speed sampling uniformly from max and min at the start of + # each rollout + "randomize_downstream_speed": True, + "max_downstream_speed": 5.0, + "min_downstream_speed": 5.0, }) -# CREATE VEHICLE TYPES AND INFLOWS -# no vehicles in the network +# =========================================================================== # +# Specify vehicle-specific information and inflows. # +# =========================================================================== # + +# create the base vehicle types that will be used for inflows vehicles = VehicleParams() -vehicles.add( - "human", - num_vehicles=0, - lane_change_params=SumoLaneChangeParams( - lane_change_mode="strategic", +if ON_RAMP: + vehicles.add( + "human", + num_vehicles=0, + routing_controller=(I210Router, {}), + acceleration_controller=(IDMController, { + 'a': 1.3, + 'b': 2.0, + 'noise': 0.3, + "fail_safe": ['obey_speed_limit', 'safe_velocity', 'feasible_accel'], + }), + car_following_params=SumoCarFollowingParams( + min_gap=0.5, + speed_mode=12 # right of way at intersections + obey limits on deceleration + ), + lane_change_params=SumoLaneChangeParams( + lane_change_mode="sumo_default", + ), ) -) +else: + vehicles.add( + "human", + num_vehicles=0, + acceleration_controller=(IDMController, { + 'a': 1.3, + 'b': 2.0, + 'noise': 0.3, + "fail_safe": ['obey_speed_limit', 'safe_velocity', 'feasible_accel'], + }), + car_following_params=SumoCarFollowingParams( + min_gap=0.5, + speed_mode=12 # right of way at intersections + obey limits on deceleration + ), + lane_change_params=SumoLaneChangeParams( + lane_change_mode="sumo_default", + ), + ) + +# autonomous vehicles vehicles.add( - "av", - acceleration_controller=(RLController, {}), - num_vehicles=0, -) + color='red', + veh_id='rl', + car_following_params=SumoCarFollowingParams( + min_gap=0.5, + speed_mode=12 # right of way at intersections + obey limits on deceleration + ), + acceleration_controller=(RLController, { + "fail_safe": ['obey_speed_limit', 'safe_velocity', 'feasible_accel'], + })) + inflow = InFlows() -# main highway -pen_rate = PENETRATION_RATE / 100 -assert pen_rate < 1.0, "your penetration rate is over 100%" -assert pen_rate > 0.0, "your penetration rate should be above zero" -inflow.add( - veh_type="human", - edge="119257914", - vehs_per_hour=8378 * pen_rate, - # probability=1.0, - departLane="random", - departSpeed=20) -# on ramp -# inflow.add( -# veh_type="human", -# edge="27414345", -# vehs_per_hour=321 * pen_rate, -# departLane="random", -# departSpeed=20) -# inflow.add( -# veh_type="human", -# edge="27414342#0", -# vehs_per_hour=421 * pen_rate, -# departLane="random", -# departSpeed=20) - -# Now add the AVs -# main highway -inflow.add( - veh_type="av", - edge="119257914", - vehs_per_hour=int(8378 * pen_rate), - # probability=1.0, - departLane="random", - departSpeed=20) -# # on ramp -# inflow.add( -# veh_type="av", -# edge="27414345", -# vehs_per_hour=int(321 * pen_rate), -# departLane="random", -# departSpeed=20) -# inflow.add( -# veh_type="av", -# edge="27414342#0", -# vehs_per_hour=int(421 * pen_rate), -# departLane="random", -# departSpeed=20) - -NET_TEMPLATE = os.path.join( - config.PROJECT_PATH, - "examples/exp_configs/templates/sumo/test2.net.xml") +if ENTER_AS_LINE: + for lane in [0, 1, 2, 3, 4]: + if WANT_BOUNDARY_CONDITIONS: + # Add the inflows from the main highway. + inflow.add( + veh_type="human", + edge="ghost0", + vehs_per_hour=int(INFLOW_RATE * (1 - PENETRATION_RATE)), + departLane=lane, + departSpeed=INFLOW_SPEED) + inflow.add( + veh_type="rl", + edge="ghost0", + vehs_per_hour=int(INFLOW_RATE * PENETRATION_RATE), + departLane=lane, + departSpeed=INFLOW_SPEED) + else: + # Add the inflows from the main highway. + inflow.add( + veh_type="human", + edge="119257914", + vehs_per_hour=int(INFLOW_RATE * (1 - PENETRATION_RATE)), + departLane=lane, + departSpeed=INFLOW_SPEED) + inflow.add( + veh_type="rl", + edge="119257914", + vehs_per_hour=int(INFLOW_RATE * PENETRATION_RATE), + departLane=lane, + departSpeed=INFLOW_SPEED) + + # Add the inflows from the on-ramps. + if ON_RAMP: + inflow.add( + veh_type="human", + edge="27414345", + vehs_per_hour=int(ON_RAMP_INFLOW_RATE * (1 - PENETRATION_RATE)), + departLane="random", + departSpeed=10) + inflow.add( + veh_type="human", + edge="27414342#0", + vehs_per_hour=int(ON_RAMP_INFLOW_RATE * (1 - PENETRATION_RATE)), + departLane="random", + departSpeed=10) +else: + if WANT_BOUNDARY_CONDITIONS: + # Add the inflows from the main highway. + inflow.add( + veh_type="human", + edge="ghost0", + vehs_per_hour=int(INFLOW_RATE * 5 * (1 - PENETRATION_RATE)), + departLane="best", + departSpeed=INFLOW_SPEED) + inflow.add( + veh_type="rl", + edge="ghost0", + vehs_per_hour=int(INFLOW_RATE * 5 * PENETRATION_RATE), + departLane="best", + departSpeed=INFLOW_SPEED) + else: + # Add the inflows from the main highway. + inflow.add( + veh_type="human", + edge="119257914", + vehs_per_hour=int(INFLOW_RATE * 5 * (1 - PENETRATION_RATE)), + departLane="best", + departSpeed=INFLOW_SPEED) + inflow.add( + veh_type="rl", + edge="119257914", + vehs_per_hour=int(INFLOW_RATE * 5 * PENETRATION_RATE), + departLane="best", + departSpeed=INFLOW_SPEED) + + # Add the inflows from the on-ramps. + if ON_RAMP: + inflow.add( + veh_type="human", + edge="27414345", + vehs_per_hour=int(ON_RAMP_INFLOW_RATE * (1 - PENETRATION_RATE)), + departLane="best", + departSpeed=10) + inflow.add( + veh_type="human", + edge="27414342#0", + vehs_per_hour=int(ON_RAMP_INFLOW_RATE * (1 - PENETRATION_RATE)), + departLane="best", + departSpeed=10) + +# =========================================================================== # +# Generate the flow_params dict with all relevant simulation information. # +# =========================================================================== # flow_params = dict( # name of the experiment @@ -132,17 +297,22 @@ # simulation-related parameters sim=SumoParams( - sim_step=0.8, + sim_step=0.4, render=False, - color_by_speed=True, - restart_instance=True + color_by_speed=False, + restart_instance=True, + use_ballistic=True, + disable_collisions=True, + overtake_right=True, ), # environment related parameters (see flow.core.params.EnvParams) env=EnvParams( horizon=HORIZON, - sims_per_step=1, + sims_per_step=3, + warmup_steps=WARMUP_STEPS, additional_params=additional_env_params, + done_at_exit=not additional_env_params["reroute_on_exit"] ), # network-related parameters (see flow.core.params.NetParams and the @@ -151,8 +321,8 @@ inflows=inflow, template=NET_TEMPLATE, additional_params={ - "on_ramp": False, - "ghost_edge": False + "on_ramp": ON_RAMP, + "ghost_edge": WANT_BOUNDARY_CONDITIONS } ), @@ -167,19 +337,21 @@ ), ) -# SET UP RLLIB MULTI-AGENT FEATURES +# =========================================================================== # +# Set up rllib multi-agent features. # +# =========================================================================== # create_env, env_name = make_create_env(params=flow_params, version=0) # register as rllib env register_env(env_name, create_env) -# multiagent configuration +# multi-agent configuration test_env = create_env() obs_space = test_env.observation_space act_space = test_env.action_space -POLICY_GRAPHS = {'av': (PPOTFPolicy, obs_space, act_space, {})} +POLICY_GRAPHS = {'av': (None, obs_space, act_space, {})} POLICIES_TO_TRAIN = ['av'] @@ -187,3 +359,14 @@ def policy_mapping_fn(_): """Map a policy in RLlib.""" return 'av' + + +custom_callables = { + "avg_speed": lambda env: np.mean([speed for speed in + env.k.vehicle.get_speed(env.k.vehicle.get_ids()) if speed >= 0]), + "avg_outflow": lambda env: np.nan_to_num(env.k.vehicle.get_outflow_rate(120)), + "avg_energy": lambda env: -1 * energy_consumption(env, 0.1), + "avg_per_step_energy": lambda env: -1 * energy_consumption(env, 0.1) / env.k.vehicle.num_vehicles + if env.k.vehicle.num_vehicles > 0 + else 0, +} diff --git a/examples/exp_configs/rl/multiagent/multiagent_merge.py b/examples/exp_configs/rl/multiagent/multiagent_merge.py index bfc9fb3b7..312d3a0dd 100644 --- a/examples/exp_configs/rl/multiagent/multiagent_merge.py +++ b/examples/exp_configs/rl/multiagent/multiagent_merge.py @@ -3,7 +3,7 @@ Trains a a small percentage of rl vehicles to dissipate shockwaves caused by on-ramp merge to a single lane open highway network. """ -from ray.rllib.agents.ppo.ppo_policy import PPOTFPolicy +from ray.rllib.agents.ppo.ppo_tf_policy import PPOTFPolicy from ray.tune.registry import register_env from flow.core.params import SumoParams, EnvParams, InitialConfig diff --git a/examples/exp_configs/rl/multiagent/multiagent_ring.py b/examples/exp_configs/rl/multiagent/multiagent_ring.py index a789174f4..c2fc52f14 100644 --- a/examples/exp_configs/rl/multiagent/multiagent_ring.py +++ b/examples/exp_configs/rl/multiagent/multiagent_ring.py @@ -3,7 +3,7 @@ Trains a number of autonomous vehicles to stabilize the flow of 22 vehicles in a variable length ring road. """ -from ray.rllib.agents.ppo.ppo_policy import PPOTFPolicy +from ray.rllib.agents.ppo.ppo_tf_policy import PPOTFPolicy from ray.tune.registry import register_env from flow.core.params import SumoParams, EnvParams, InitialConfig, NetParams diff --git a/examples/exp_configs/rl/multiagent/multiagent_straight_road.py b/examples/exp_configs/rl/multiagent/multiagent_straight_road.py new file mode 100644 index 000000000..bf37beb71 --- /dev/null +++ b/examples/exp_configs/rl/multiagent/multiagent_straight_road.py @@ -0,0 +1,221 @@ +"""Multi-agent highway with ramps example. + +Trains a non-constant number of agents, all sharing the same policy, on the +highway with ramps network. +""" +from flow.controllers import RLController, IDMController +from flow.core.params import EnvParams, NetParams, InitialConfig, InFlows, \ + VehicleParams, SumoParams, SumoCarFollowingParams +from flow.networks import HighwayNetwork +from flow.envs.ring.accel import ADDITIONAL_ENV_PARAMS +from flow.envs.multiagent import MultiStraightRoad +from flow.networks.highway import ADDITIONAL_NET_PARAMS +from flow.utils.registry import make_create_env +from ray.tune.registry import register_env + +# SET UP PARAMETERS FOR THE SIMULATION + +# the speed of vehicles entering the network +TRAFFIC_SPEED = 24.1 +# the maximum speed at the downstream boundary edge +END_SPEED = 6.0 +# the inflow rate of vehicles +HIGHWAY_INFLOW_RATE = 2215 +# the simulation time horizon (in steps) +HORIZON = 1000 +# whether to include noise in the car-following models +INCLUDE_NOISE = True + +PENETRATION_RATE = 10.0 + +WARMUP_STEPS = 500 + +additional_net_params = ADDITIONAL_NET_PARAMS.copy() +additional_net_params.update({ + # length of the highway + "length": 2500, + # number of lanes + "lanes": 1, + # speed limit for all edges + "speed_limit": 30, + # number of edges to divide the highway into + "num_edges": 2, + # whether to include a ghost edge + "use_ghost_edge": True, + # speed limit for the ghost edge + "ghost_speed_limit": END_SPEED, + # length of the cell imposing a boundary + "boundary_cell_length": 300, +}) + + +# SET UP PARAMETERS FOR THE ENVIRONMENT + +additional_env_params = ADDITIONAL_ENV_PARAMS.copy() +additional_env_params.update({ + 'max_accel': 2.6, + 'max_decel': 4.5, + 'target_velocity': 6.0, + 'local_reward': True, + 'lead_obs': True, + 'control_range': [500, 2300], + # whether to reroute vehicles once they have exited + "reroute_on_exit": False, + # whether to use the MPG reward. Otherwise, defaults to a target velocity reward + "mpg_reward": False, + # whether to use the joules reward. Otherwise, defaults to a target velocity reward + "mpj_reward": False, + # how many vehicles to look back for the MPG reward + "look_back_length": 3, + # how many AVs there can be at once (this is only for centralized critics) + "max_num_agents": 10, + + # whether to add a slight reward for opening up a gap that will be annealed out N iterations in + "headway_curriculum": False, + # how many timesteps to anneal the headway curriculum over + "headway_curriculum_iters": 100, + # weight of the headway reward + "headway_reward_gain": 2.0, + # desired time headway + "min_time_headway": 2.0, + + # whether to add a slight reward for traveling at a desired speed + "speed_curriculum": True, + # how many timesteps to anneal the headway curriculum over + "speed_curriculum_iters": 20, + # weight of the headway reward + "speed_reward_gain": 1.0, + + # penalize stopped vehicles + "penalize_stops": True, + "stop_penalty": 0.05, + + # penalize accels + "penalize_accel": True, + "accel_penalty": 0.05, + + # control the range of speeds of the downstream edge + # if true, we vary the downstream speed sampling uniformly from max and min at the start of + # each rollout + "randomize_downstream_speed": True, + "max_downstream_speed": 5.0, + "min_downstream_speed": 5.0, +}) + + +# CREATE VEHICLE TYPES AND INFLOWS + +vehicles = VehicleParams() +inflows = InFlows() +vehicles.add( + "human", + acceleration_controller=(IDMController, { + 'a': 1.3, + 'b': 2.0, + 'noise': 0.3 if INCLUDE_NOISE else 0.0, + "fail_safe": ['obey_speed_limit', 'safe_velocity', 'feasible_accel'], + }), + car_following_params=SumoCarFollowingParams( + speed_mode=12, # right of way at intersections + obey limits on deceleration + min_gap=0.5 + ), +) + +# autonomous vehicles +vehicles.add( + color='red', + veh_id='rl', + car_following_params=SumoCarFollowingParams( + min_gap=0.5, + speed_mode=12 # right of way at intersections + obey limits on deceleration + ), + acceleration_controller=(RLController, { + "fail_safe": ['obey_speed_limit', 'safe_velocity', 'feasible_accel'], + })) + +# add human vehicles on the highway +inflows.add( + veh_type="human", + edge="highway_0", + vehs_per_hour=int(HIGHWAY_INFLOW_RATE * (1 - PENETRATION_RATE / 100)), + depart_lane="free", + depart_speed=TRAFFIC_SPEED, + name="idm_highway_inflow") + +# add autonomous vehicles on the highway +# they will stay on the highway, i.e. they won't exit through the off-ramps +inflows.add( + veh_type="rl", + edge="highway_0", + vehs_per_hour=int(HIGHWAY_INFLOW_RATE * (PENETRATION_RATE / 100)), + depart_lane="free", + depart_speed=TRAFFIC_SPEED, + name="rl_highway_inflow") + +flow_params = dict( + # name of the experiment + exp_tag='multiagent_highway', + + # name of the flow environment the experiment is running on + env_name=MultiStraightRoad, + + # name of the network class the experiment is running on + network=HighwayNetwork, + + # simulator that is used by the experiment + simulator='traci', + + # environment related parameters (see flow.core.params.EnvParams) + env=EnvParams( + horizon=HORIZON, + warmup_steps=WARMUP_STEPS, + sims_per_step=3, + additional_params=additional_env_params + ), + + # sumo-related parameters (see flow.core.params.SumoParams) + sim=SumoParams( + sim_step=0.4, + render=False, + restart_instance=True, + use_ballistic=True + ), + + # network-related parameters (see flow.core.params.NetParams and the + # network's documentation or ADDITIONAL_NET_PARAMS component) + net=NetParams( + inflows=inflows, + additional_params=additional_net_params + ), + + # vehicles to be placed in the network at the start of a rollout (see + # flow.core.params.VehicleParams) + veh=vehicles, + + # parameters specifying the positioning of vehicles upon initialization/ + # reset (see flow.core.params.InitialConfig) + initial=InitialConfig(), +) + + +# SET UP RLLIB MULTI-AGENT FEATURES + +create_env, env_name = make_create_env(params=flow_params, version=0) + +# register as rllib env +register_env(env_name, create_env) + +# multiagent configuration +test_env = create_env() +obs_space = test_env.observation_space +act_space = test_env.action_space + + +POLICY_GRAPHS = {'av': (None, obs_space, act_space, {})} + +POLICIES_TO_TRAIN = ['av'] + + +def policy_mapping_fn(_): + """Map a policy in RLlib.""" + return 'av' diff --git a/examples/exp_configs/rl/multiagent/multiagent_traffic_light_grid.py b/examples/exp_configs/rl/multiagent/multiagent_traffic_light_grid.py index b8293f638..308dfa0d7 100644 --- a/examples/exp_configs/rl/multiagent/multiagent_traffic_light_grid.py +++ b/examples/exp_configs/rl/multiagent/multiagent_traffic_light_grid.py @@ -1,6 +1,6 @@ """Multi-agent traffic light example (single shared policy).""" -from ray.rllib.agents.ppo.ppo_policy import PPOTFPolicy +from ray.rllib.agents.ppo.ppo_tf_policy import PPOTFPolicy from flow.envs.multiagent import MultiTrafficLightGridPOEnv from flow.networks import TrafficLightGridNetwork from flow.core.params import SumoParams, EnvParams, InitialConfig, NetParams diff --git a/examples/exp_configs/rl/singleagent/singleagent_straight_road.py b/examples/exp_configs/rl/singleagent/singleagent_straight_road.py new file mode 100644 index 000000000..e5312a7f9 --- /dev/null +++ b/examples/exp_configs/rl/singleagent/singleagent_straight_road.py @@ -0,0 +1,179 @@ +"""Multi-agent highway with ramps example. + +Trains a non-constant number of agents, all sharing the same policy, on the +highway with ramps network. +""" +from flow.controllers import RLController, IDMController +from flow.core.params import EnvParams, NetParams, InitialConfig, InFlows, \ + VehicleParams, SumoParams, SumoLaneChangeParams, SumoCarFollowingParams +from flow.envs.ring.accel import ADDITIONAL_ENV_PARAMS +from flow.networks import HighwayNetwork +from flow.envs import SingleStraightRoad +from flow.networks.highway import ADDITIONAL_NET_PARAMS +from flow.utils.registry import make_create_env +from ray.tune.registry import register_env + +# SET UP PARAMETERS FOR THE SIMULATION + +# number of steps per rollout +HORIZON = 2000 + +# inflow rate on the highway in vehicles per hour +HIGHWAY_INFLOW_RATE = 10800 / 5 +# percentage of autonomous vehicles compared to human vehicles on highway +PENETRATION_RATE = 10 + +# SET UP PARAMETERS FOR THE NETWORK + +additional_net_params = ADDITIONAL_NET_PARAMS.copy() +additional_net_params.update({ + # length of the highway + "length": 2000, + # number of lanes + "lanes": 1, + # speed limit for all edges + "speed_limit": 30, + # number of edges to divide the highway into + "num_edges": 2 +}) + +# SET UP PARAMETERS FOR THE ENVIRONMENT + +additional_env_params = ADDITIONAL_ENV_PARAMS.copy() +additional_env_params.update({ + 'max_accel': 2.6, + 'max_decel': 4.5, + 'target_velocity': 18.0, + 'local_reward': True, + 'lead_obs': True, + "terminate_on_wave": False, + # the environment is not allowed to terminate below this horizon length + 'wave_termination_horizon': 1000, + # the speed below which we consider a wave to have occured + 'wave_termination_speed': 10.0, + # whether the vehicle continues to acquire reward after it exits the system. This causes it to have incentive + # to leave the network in a good state after it leaves + 'reward_after_exit': True +}) + +# CREATE VEHICLE TYPES AND INFLOWS + +vehicles = VehicleParams() +inflows = InFlows() + +# human vehicles +vehicles.add( + "human", + num_vehicles=0, + lane_change_params=SumoLaneChangeParams( + lane_change_mode="strategic", + ), + car_following_params=SumoCarFollowingParams( + speed_mode=12, + min_gap=0.5 + ), + acceleration_controller=(IDMController, { + "a": .3, + "b": 2.0, + "noise": 0.5, + "fail_safe": [ + 'obey_speed_limit', + 'safe_velocity', + 'feasible_accel', + ], + }), +) + +# autonomous vehicles +vehicles.add( + color='red', + veh_id='rl', + car_following_params=SumoCarFollowingParams( + min_gap=0.5, + speed_mode=12 # right of way at intersections + obey limits on deceleration + ), + acceleration_controller=(RLController, { + "fail_safe": ['obey_speed_limit', 'safe_velocity', 'feasible_accel'], + })) + +# add human vehicles on the highway +inflows.add( + veh_type="human", + edge="highway_0", + vehs_per_hour=int(HIGHWAY_INFLOW_RATE * (1 - PENETRATION_RATE / 100)), + depart_lane="free", + depart_speed="23.0", + name="idm_highway_inflow") + +# add autonomous vehicles on the highway +# they will stay on the highway, i.e. they won't exit through the off-ramps +inflows.add( + veh_type="rl", + edge="highway_0", + vehs_per_hour=int(HIGHWAY_INFLOW_RATE * (PENETRATION_RATE / 100)), + depart_lane="free", + depart_speed="23.0", + name="rl_highway_inflow") + +# SET UP FLOW PARAMETERS +done_at_exit = True +if additional_env_params['reward_after_exit']: + done_at_exit = False + +flow_params = dict( + # name of the experiment + exp_tag='singleagent_highway', + + # name of the flow environment the experiment is running on + env_name=SingleStraightRoad, + + # name of the network class the experiment is running on + network=HighwayNetwork, + + # simulator that is used by the experiment + simulator='traci', + + # environment related parameters (see flow.core.params.EnvParams) + env=EnvParams( + horizon=HORIZON, + warmup_steps=500, + sims_per_step=1, # do not put more than one + done_at_exit=done_at_exit, + additional_params=additional_env_params, + ), + + # sumo-related parameters (see flow.core.params.SumoParams) + sim=SumoParams( + sim_step=0.5, + render=False, + use_ballistic=True, + restart_instance=True + ), + + # network-related parameters (see flow.core.params.NetParams and the + # network's documentation or ADDITIONAL_NET_PARAMS component) + net=NetParams( + inflows=inflows, + additional_params=additional_net_params + ), + + # vehicles to be placed in the network at the start of a rollout (see + # flow.core.params.VehicleParams) + veh=vehicles, + + # parameters specifying the positioning of vehicles upon initialization/ + # reset (see flow.core.params.InitialConfig) + initial=InitialConfig(), +) + +# SET UP RLLIB MULTI-AGENT FEATURES + +create_env, env_name = make_create_env(params=flow_params, version=0) + +# register as rllib env +register_env(env_name, create_env) + +# multiagent configuration +test_env = create_env() +obs_space = test_env.observation_space +act_space = test_env.action_space diff --git a/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml b/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml index ee508b730..f337e9440 100644 --- a/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml +++ b/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml @@ -2392,23 +2392,23 @@ - - + + - + - + - + - + - + diff --git a/examples/exp_scripts/energy_bracketing.py b/examples/exp_scripts/energy_bracketing.py new file mode 100644 index 000000000..3902d7a06 --- /dev/null +++ b/examples/exp_scripts/energy_bracketing.py @@ -0,0 +1,227 @@ +"""Energy bracketing runner script for computing min/max energy savings bounds.""" +from flow.networks import HighwayNetwork +from flow.core.params import NetParams, InitialConfig +from flow.networks.highway import ADDITIONAL_NET_PARAMS +from flow.envs import SingleStraightRoadEnergyBracketing +from flow.envs.straightroad_env import ADDITIONAL_ENV_PARAMS +from flow.core.params import VehicleParams +from flow.controllers import IDMController +from flow.controllers import TrajectoryFollower +from flow.core.params import SumoParams +from flow.core.params import EnvParams, SumoCarFollowingParams +from flow.data_pipeline.data_pipeline import write_dict_to_csv, get_extra_info +import argparse +from collections import defaultdict +import uuid +import numpy as np +import os +from flow.utils.registry import make_create_env +import sys + +ROAD_LENGTH = 5000 +INITIAL_SPEED = 12.0 +N_HUMANS = 9 + + +def parse_args(args): + """Parse experiment options user can specify in command line. + + Returns + ------- + argparse.Namespace + the output parser object + """ + parser = argparse.ArgumentParser( + description="Parse argument used when running a min/max simulation.", + epilog="python min_max_exp.py --start_amp FLOAT --end_amp FLOAT --freq FLOAT --sweep_step FLOAT --emodel STR") + + parser.add_argument( + '--start_amp', type=float, default=0, + help='Starting amplitude to sweep over. Defaults to 0.') + parser.add_argument( + '--end_amp', type=float, default=15, + help='Ending amplitude to sweep over. Defaults to 15.') + parser.add_argument( + '--sweep_step', type=float, default=1, + help='Step between amplitudes. Defaults to 1.') + parser.add_argument( + '--freq', type=float, default=0.1, + help='Frequency of the sine wave. Defaults to 0.2.') + parser.add_argument( + '--emodel', type=str, + help='Energy model to be used. choose from: PriusEnergy, TacomaEnergy, PDMCombustionEngine, PDMElectric') + + return parser.parse_known_args(args)[0] + + +if __name__ == "__main__": + # Get Arguments from parser + args = parse_args(sys.argv[1:]) + # assign arguments to variables + if args.start_amp: + s_amp = args.start_amp + e_amp = args.end_amp + else: + s_amp = args.start_amp + e_amp = args.end_amp + if args.sweep_step: + sweep_step = args.sweep_step + else: + sweep_step = 1 + if args.freq: + freq = args.freq + else: + freq = 0.1 + if args.emodel: + if args.emodel == 'PriusEnergy': + from flow.energy_models.toyota_energy import PriusEnergy + + ENERGY_MODEL = PriusEnergy + elif args.emodel == 'TacomaEnergy': + from flow.energy_models.toyota_energy import TacomaEnergy + + ENERGY_MODEL = TacomaEnergy + elif args.emodel == 'PDMElectric': + from flow.energy_models.power_demand import PDMElectric + + ENERGY_MODEL = PDMElectric + elif args.emodel == 'PDMCombustionEngine': + from flow.energy_models.power_demand import PDMCombustionEngine + + ENERGY_MODEL = PDMCombustionEngine + else: + print('Energy model undefined. PDMCombustionEngine assigned by default') + from flow.energy_models.power_demand import PDMCombustionEngine + + ENERGY_MODEL = PDMCombustionEngine + + network_name = HighwayNetwork + # name of the network + name = "bracketing_example" + + # network-specific parameters + net_params = NetParams(additional_params=ADDITIONAL_NET_PARAMS) + net_params.additional_params['length'] = ROAD_LENGTH + net_params.additional_params['ghost_speed_limit'] = 30.0 + # initial configuration to vehicles + initial_config = InitialConfig() + # Place vehicle at the start of the lane with 15 m in between + initial_config.spacing = "custom" + initial_pos = {"start_positions": [], + "start_lanes": []} + for i in range(N_HUMANS + 1): + initial_pos["start_positions"].append(('highway_0', i * 20)) + initial_pos["start_lanes"].append(0) + + initial_config.additional_params = initial_pos + + # Add vehicles + vehicles = VehicleParams() + vehicles.add("human", + acceleration_controller=(IDMController, {}), + car_following_params=SumoCarFollowingParams(accel=4.0), + energy_model=ENERGY_MODEL, + initial_speed=INITIAL_SPEED, + num_vehicles=N_HUMANS) + # accel is the maximum acceleration allowed, decel default value is 4.5 + cruse_speed = INITIAL_SPEED + vehicles.add(veh_id="AV", + acceleration_controller=(TrajectoryFollower, {"func": lambda _: None}), + car_following_params=SumoCarFollowingParams(accel=4.0), + # energy_model=ENERGY_MODEL, + initial_speed=INITIAL_SPEED, + num_vehicles=1) + # get simulation parameters + sim_params = SumoParams(sim_step=0.1, render=False) + setattr(sim_params, 'num_clients', 1) + sim_params.restart_instance = True + # Set time horizon of the experiment + # (make sure it's large enough for all veh to exit) + HORIZON = 100000 + # set environment parameters + additional_env_params = ADDITIONAL_ENV_PARAMS + additional_env_params['max_accel'] = 4.0 + additional_env_params['max_decel'] = 4.0 + env_params = EnvParams( + horizon=HORIZON, + additional_params=additional_env_params + ) + # choose environment (check definition from imports) + env_name = SingleStraightRoadEnergyBracketing + # log experiment parameters + flow_params = dict( + # name of the experiment + exp_tag=name, + # name of the flow environment the experiment is running on + env_name=env_name, + # name of the network class the experiment uses + network=network_name, + # simulator that is used by the experiment + simulator='traci', + # simulation-related parameters + sim=sim_params, + # environment related parameters (see flow.core.params.EnvParams) + env=env_params, + # network-related parameters (see flow.core.params.NetParams and + # the network's documentation or ADDITIONAL_NET_PARAMS component) + net=net_params, + # vehicles to be placed in the network at the start of a rollout + # (see flow.core.vehicles.Vehicles) + veh=vehicles, + # (optional) parameters affecting the positioning of vehicles upon + # initialization/reset (see flow.core.params.InitialConfig) + initial=initial_config + ) + # create the environment + create_env, gym_name = make_create_env(params=flow_params, version=0) + env = create_env(gym_name) + + env_params.restart_instance = False + # initiate data logging + extra_info = defaultdict(lambda: []) + source_id = 'flow_{}'.format(uuid.uuid4().hex) + + rets = [] # returns = [total fuel at each run] + ftimes = [] # final times = [total time for each run] + amplitudes = list(np.arange(s_amp, e_amp + sweep_step, sweep_step)) # amplitudes used for sweeping + print("starting experiment with energy model {}".format(ENERGY_MODEL)) + for amplit in amplitudes: + run_id = "run_amp_{}".format(amplit) + env.pipeline_params = (extra_info, source_id, run_id) # update run_id + state = env.reset() # reset env at each run + ret = 0 + rl_speeds = [] # speeds of controlled vehicle as outputted from simulator + d_speeds = [] # desired speeds + # update the speed function for each run + env.k.vehicle.get_acc_controller('AV_0').speed_func = \ + lambda x: cruse_speed + amplit * np.sin(freq * (env.time_counter + 1) * env.sim_step) + # Simulate Run + for _ in range(env_params.horizon): + + step_vehicles = env.unwrapped.k.vehicle + state, reward, done, _ = env.step(None) + + # collect data for data pipeline + get_extra_info(step_vehicles, extra_info, step_vehicles.get_ids(), source_id, run_id) + + ret += reward + if done: + break + # log total fuel and time after run is done + rets.append(ret) + ftime = env.time_counter * env.sim_step + ftimes.append(ftime) + print('Amplitide {}, Fuel consumed: {}, Time taken: {}'.format(amplit, ret, ftime)) + + # terminate env after sweeping iver all amplitudes + env.unwrapped.terminate() + # dump to csv + dir_path = os.path.dirname(os.path.realpath('__file__')) + trajectory_table_path = os.path.join(dir_path, '{}_trajectories.csv'.format(source_id)) + write_dict_to_csv(trajectory_table_path, extra_info, True) + trajectory_table_path2 = trajectory_table_path[:-4] + '_amp_vs_fuel.csv' + fuel_data = defaultdict(lambda: []) + fuel_data['Ampiltude'] = amplitudes + fuel_data['Fuel'] = rets + fuel_data['Time'] = ftimes + write_dict_to_csv(trajectory_table_path2, fuel_data, True) diff --git a/examples/simulate.py b/examples/simulate.py index d1dcc5a79..1ab89837e 100644 --- a/examples/simulate.py +++ b/examples/simulate.py @@ -12,6 +12,8 @@ from flow.core.params import AimsunParams from flow.utils.rllib import FlowParamsEncoder +from flow.data_pipeline.data_pipeline import collect_metadata_from_config + def parse_args(args): """Parse training options user can specify in command line. @@ -49,6 +51,17 @@ def parse_args(args): action='store_true', help='Specifies whether to generate an emission file from the ' 'simulation.') + parser.add_argument( + '--to_aws', + type=str, nargs='?', default=None, const="default", + help='Specifies the name of the partition to store the output' + 'file on S3. Putting not None value for this argument' + 'automatically set gen_emission to True.') + parser.add_argument( + '--is_baseline', + action='store_true', + help='specifies whether this is a baseline run' + ) return parser.parse_known_args(args)[0] @@ -56,15 +69,18 @@ def parse_args(args): if __name__ == "__main__": flags = parse_args(sys.argv[1:]) + flags.gen_emission = flags.gen_emission or flags.to_aws + # Get the flow_params object. module = __import__("exp_configs.non_rl", fromlist=[flags.exp_config]) - flow_params = getattr(module, flags.exp_config).flow_params + config_obj = getattr(module, flags.exp_config) + flow_params = config_obj.flow_params # Get the custom callables for the runner. - if hasattr(getattr(module, flags.exp_config), "custom_callables"): - callables = getattr(module, flags.exp_config).custom_callables - else: - callables = None + callables = getattr(config_obj, "custom_callables", None) + + # load some metadata from the exp_config file + supplied_metadata = collect_metadata_from_config(config_obj) flow_params['sim'].render = not flags.no_render flow_params['simulator'] = 'aimsun' if flags.aimsun else 'traci' @@ -90,4 +106,5 @@ def parse_args(args): exp = Experiment(flow_params, callables) # Run for the specified number of rollouts. - exp.run(flags.num_runs, convert_to_csv=flags.gen_emission) + exp.run(flags.num_runs, convert_to_csv=flags.gen_emission, to_aws=flags.to_aws, + is_baseline=flags.is_baseline, supplied_metadata=supplied_metadata) diff --git a/examples/train.py b/examples/train.py index 5f8edbb22..2d2a17aea 100644 --- a/examples/train.py +++ b/examples/train.py @@ -7,16 +7,21 @@ python train.py EXP_CONFIG """ import argparse +from datetime import datetime import json import os import sys from time import strftime from copy import deepcopy +import numpy as np +import pytz from flow.core.util import ensure_dir +from flow.core.rewards import instantaneous_mpg from flow.utils.registry import env_constructor from flow.utils.rllib import FlowParamsEncoder, get_flow_params from flow.utils.registry import make_create_env +from flow.replay.transfer_tests import create_parser, generate_graphs def parse_args(args): @@ -42,19 +47,52 @@ def parse_args(args): parser.add_argument( '--rl_trainer', type=str, default="rllib", help='the RL trainer to use. either rllib or Stable-Baselines') - + parser.add_argument( + '--algorithm', type=str, default="PPO", + help='RL algorithm to use. Options are PPO, TD3, and CENTRALIZEDPPO (which uses a centralized value function)' + ' right now.' + ) parser.add_argument( '--num_cpus', type=int, default=1, help='How many CPUs to use') parser.add_argument( '--num_steps', type=int, default=5000, - help='How many total steps to perform learning over') + help='How many total steps to perform learning over. Relevant for stable-baselines') + parser.add_argument( + '--grid_search', action='store_true', default=False, + help='Whether to grid search over hyperparams') + parser.add_argument( + '--num_iterations', type=int, default=200, + help='How many iterations are in a training run.') + parser.add_argument( + '--checkpoint_freq', type=int, default=20, + help='How often to checkpoint.') + parser.add_argument( + '--num_rollouts', type=int, default=1, + help='How many rollouts are in a training batch') parser.add_argument( '--rollout_size', type=int, default=1000, help='How many steps are in a training batch.') + parser.add_argument('--use_s3', action='store_true', default=False, + help='If true, upload results to s3') + parser.add_argument('--local_mode', action='store_true', default=False, + help='If true only 1 CPU will be used') + parser.add_argument('--render', action='store_true', default=False, + help='If true, we render the display') parser.add_argument( '--checkpoint_path', type=str, default=None, help='Directory with checkpoint to restore training from.') + parser.add_argument( + '--exp_title', type=str, default=None, + help='Name of experiment that results will be stored in') + parser.add_argument('--multi_node', action='store_true', + help='Set to true if this will be run in cluster mode.' + 'Relevant for rllib') + parser.add_argument( + '--upload_graphs', type=str, nargs=2, + help='Whether to generate and upload graphs to leaderboard at the end of training.' + 'Arguments are name of the submitter and name of the strategy.' + 'Only relevant for i210 training on rllib') return parser.parse_known_args(args)[0] @@ -84,7 +122,6 @@ def run_model_stablebaseline(flow_params, """ from stable_baselines.common.vec_env import DummyVecEnv, SubprocVecEnv from stable_baselines import PPO2 - if num_cpus == 1: constructor = env_constructor(params=flow_params, version=0)() # The algorithms require a vectorized environment to run @@ -101,9 +138,11 @@ def run_model_stablebaseline(flow_params, def setup_exps_rllib(flow_params, n_cpus, n_rollouts, + flags, policy_graphs=None, policy_mapping_fn=None, - policies_to_train=None): + policies_to_train=None, + ): """Return the relevant components of an RLlib experiment. Parameters @@ -114,13 +153,14 @@ def setup_exps_rllib(flow_params, number of CPUs to run the experiment over n_rollouts : int number of rollouts per training iteration + flags : TODO + custom arguments policy_graphs : dict, optional TODO policy_mapping_fn : function, optional TODO policies_to_train : list of str, optional TODO - Returns ------- str @@ -132,6 +172,7 @@ def setup_exps_rllib(flow_params, """ from ray import tune from ray.tune.registry import register_env + from ray.rllib.env.group_agents_wrapper import _GroupAgentsWrapper try: from ray.rllib.agents.agent import get_agent_class except ImportError: @@ -139,20 +180,141 @@ def setup_exps_rllib(flow_params, horizon = flow_params['env'].horizon - alg_run = "PPO" - - agent_cls = get_agent_class(alg_run) - config = deepcopy(agent_cls._default_config) + alg_run = flags.algorithm.upper() + + if alg_run == "PPO": + from flow.algorithms.custom_ppo import CustomPPOTrainer + from ray.rllib.agents.ppo import DEFAULT_CONFIG + alg_run = CustomPPOTrainer + config = deepcopy(DEFAULT_CONFIG) + + config["num_workers"] = n_cpus + config["horizon"] = horizon + config["model"].update({"fcnet_hiddens": [32, 32]}) + config["train_batch_size"] = horizon * n_rollouts + config["gamma"] = 0.995 # discount rate + config["use_gae"] = True + config["no_done_at_end"] = True + config["lambda"] = 0.97 + config["kl_target"] = 0.02 + config["num_sgd_iter"] = 10 + if flags.grid_search: + config["lambda"] = tune.grid_search([0.5, 0.9]) + config["lr"] = tune.grid_search([5e-4, 5e-5]) + elif alg_run == "CENTRALIZEDPPO": + from flow.algorithms.centralized_PPO import CCTrainer, CentralizedCriticModel + from ray.rllib.agents.ppo import DEFAULT_CONFIG + from ray.rllib.models import ModelCatalog + alg_run = CCTrainer + config = deepcopy(DEFAULT_CONFIG) + config['model']['custom_model'] = "cc_model" + config["model"]["custom_options"]["max_num_agents"] = flow_params['env'].additional_params['max_num_agents'] + config["model"]["custom_options"]["central_vf_size"] = 100 + + ModelCatalog.register_custom_model("cc_model", CentralizedCriticModel) + + config["num_workers"] = n_cpus + config["horizon"] = horizon + config["model"].update({"fcnet_hiddens": [32, 32]}) + config["train_batch_size"] = horizon * n_rollouts + config["gamma"] = 0.995 # discount rate + config["use_gae"] = True + config["lambda"] = 0.97 + config["kl_target"] = 0.02 + config["num_sgd_iter"] = 10 + if flags.grid_search: + config["lambda"] = tune.grid_search([0.5, 0.9]) + config["lr"] = tune.grid_search([5e-4, 5e-5]) + + elif alg_run == "TD3": + alg_run = get_agent_class(alg_run) + config = deepcopy(alg_run._default_config) + + config["num_workers"] = n_cpus + config["horizon"] = horizon + config["learning_starts"] = 10000 + config["buffer_size"] = 20000 # reduced to test if this is the source of memory problems + if flags.grid_search: + config["prioritized_replay"] = tune.grid_search(['True', 'False']) + config["actor_lr"] = tune.grid_search([1e-3, 1e-4]) + config["critic_lr"] = tune.grid_search([1e-3, 1e-4]) + config["n_step"] = tune.grid_search([1, 10]) - config["num_workers"] = n_cpus - config["train_batch_size"] = horizon * n_rollouts - config["gamma"] = 0.999 # discount rate - config["model"].update({"fcnet_hiddens": [32, 32, 32]}) - config["use_gae"] = True - config["lambda"] = 0.97 - config["kl_target"] = 0.02 - config["num_sgd_iter"] = 10 - config["horizon"] = horizon + else: + sys.exit("We only support PPO, TD3, right now.") + + # define some standard and useful callbacks + def on_episode_start(info): + episode = info["episode"] + episode.user_data["avg_speed"] = [] + episode.user_data["avg_speed_avs"] = [] + episode.user_data["avg_energy"] = [] + episode.user_data["inst_mpg"] = [] + episode.user_data["num_cars"] = [] + episode.user_data["avg_accel_human"] = [] + episode.user_data["avg_accel_avs"] = [] + + def on_episode_step(info): + episode = info["episode"] + env = info["env"].get_unwrapped()[0] + if isinstance(env, _GroupAgentsWrapper): + env = env.env + if hasattr(env, 'no_control_edges'): + veh_ids = [ + veh_id for veh_id in env.k.vehicle.get_ids() + if env.k.vehicle.get_speed(veh_id) >= 0 + and env.k.vehicle.get_edge(veh_id) not in env.no_control_edges + ] + rl_ids = [ + veh_id for veh_id in env.k.vehicle.get_rl_ids() + if env.k.vehicle.get_speed(veh_id) >= 0 + and env.k.vehicle.get_edge(veh_id) not in env.no_control_edges + ] + else: + veh_ids = [veh_id for veh_id in env.k.vehicle.get_ids() if env.k.vehicle.get_speed(veh_id) >= 0] + rl_ids = [veh_id for veh_id in env.k.vehicle.get_rl_ids() if env.k.vehicle.get_speed(veh_id) >= 0] + + speed = np.mean([speed for speed in env.k.vehicle.get_speed(veh_ids)]) + if not np.isnan(speed): + episode.user_data["avg_speed"].append(speed) + av_speed = np.mean([speed for speed in env.k.vehicle.get_speed(rl_ids) if speed >= 0]) + if not np.isnan(av_speed): + episode.user_data["avg_speed_avs"].append(av_speed) + episode.user_data["inst_mpg"].append(instantaneous_mpg(env, veh_ids, gain=1.0)) + episode.user_data["num_cars"].append(len(env.k.vehicle.get_ids())) + episode.user_data["avg_accel_human"].append(np.nan_to_num(np.mean( + [np.abs((env.k.vehicle.get_speed(veh_id) - env.k.vehicle.get_previous_speed(veh_id))/env.sim_step) for + veh_id in veh_ids if veh_id in env.k.vehicle.previous_speeds.keys()] + ))) + episode.user_data["avg_accel_avs"].append(np.nan_to_num(np.mean( + [np.abs((env.k.vehicle.get_speed(veh_id) - env.k.vehicle.get_previous_speed(veh_id))/env.sim_step) for + veh_id in rl_ids if veh_id in env.k.vehicle.previous_speeds.keys()] + ))) + + def on_episode_end(info): + episode = info["episode"] + avg_speed = np.mean(episode.user_data["avg_speed"]) + episode.custom_metrics["avg_speed"] = avg_speed + avg_speed_avs = np.mean(episode.user_data["avg_speed_avs"]) + episode.custom_metrics["avg_speed_avs"] = avg_speed_avs + episode.custom_metrics["avg_accel_avs"] = np.mean(episode.user_data["avg_accel_avs"]) + episode.custom_metrics["avg_energy_per_veh"] = np.mean(episode.user_data["avg_energy"]) + episode.custom_metrics["avg_mpg_per_veh"] = np.mean(episode.user_data["inst_mpg"]) + episode.custom_metrics["num_cars"] = np.mean(episode.user_data["num_cars"]) + env = info["env"].get_unwrapped()[0] + episode.custom_metrics["num_lcs"] = env.k.vehicle.lane_change_count + + def on_train_result(info): + """Store the mean score of the episode, and increment or decrement the iteration number for curriculum.""" + trainer = info["trainer"] + trainer.workers.foreach_worker( + lambda ev: ev.foreach_env( + lambda env: env.set_iteration_num())) + + config["callbacks"] = {"on_episode_start": tune.function(on_episode_start), + "on_episode_step": tune.function(on_episode_step), + "on_episode_end": tune.function(on_episode_end), + "on_train_result": tune.function(on_train_result)} # save the flow params for replay flow_json = json.dumps( @@ -162,17 +324,14 @@ def setup_exps_rllib(flow_params, # multiagent configuration if policy_graphs is not None: - print("policy_graphs", policy_graphs) config['multiagent'].update({'policies': policy_graphs}) if policy_mapping_fn is not None: - config['multiagent'].update( - {'policy_mapping_fn': tune.function(policy_mapping_fn)}) + config['multiagent'].update({'policy_mapping_fn': tune.function(policy_mapping_fn)}) if policies_to_train is not None: config['multiagent'].update({'policies_to_train': policies_to_train}) create_env, gym_name = make_create_env(params=flow_params) - # Register as rllib env register_env(gym_name, create_env) return alg_run, gym_name, config @@ -180,37 +339,93 @@ def setup_exps_rllib(flow_params, def train_rllib(submodule, flags): """Train policies using the PPO algorithm in RLlib.""" import ray - from ray.tune import run_experiments + from ray import tune flow_params = submodule.flow_params - n_cpus = submodule.N_CPUS - n_rollouts = submodule.N_ROLLOUTS + flow_params['sim'].render = flags.render policy_graphs = getattr(submodule, "POLICY_GRAPHS", None) policy_mapping_fn = getattr(submodule, "policy_mapping_fn", None) policies_to_train = getattr(submodule, "policies_to_train", None) alg_run, gym_name, config = setup_exps_rllib( - flow_params, n_cpus, n_rollouts, + flow_params, flags.num_cpus, flags.num_rollouts, flags, policy_graphs, policy_mapping_fn, policies_to_train) - ray.init(num_cpus=n_cpus + 1, object_store_memory=200 * 1024 * 1024) - exp_config = { - "run": alg_run, - "env": gym_name, - "config": { - **config - }, - "checkpoint_freq": 20, + config['num_workers'] = flags.num_cpus + config['env'] = gym_name + + # create a custom string that makes looking at the experiment names easier + def trial_str_creator(trial): + return "{}_{}".format(trial.trainable_name, trial.experiment_tag) + + if flags.multi_node: + ray.init(redis_address='localhost:6379') + elif flags.local_mode: + ray.init(local_mode=True) + else: + ray.init() + exp_dict = { + "run_or_experiment": alg_run, + "name": flags.exp_title or flow_params['exp_tag'], + "config": config, + "checkpoint_freq": flags.checkpoint_freq, "checkpoint_at_end": True, - "max_failures": 999, + 'trial_name_creator': trial_str_creator, + "max_failures": 0, "stop": { - "training_iteration": flags.num_steps, + "training_iteration": flags.num_iterations, }, } - - if flags.checkpoint_path is not None: - exp_config['restore'] = flags.checkpoint_path - run_experiments({flow_params["exp_tag"]: exp_config}) + date = datetime.now(tz=pytz.utc) + date = date.astimezone(pytz.timezone('US/Pacific')).strftime("%m-%d-%Y") + if flags.use_s3: + s3_string = "s3://i210.experiments/i210/" \ + + date + '/' + flags.exp_title + exp_dict['upload_dir'] = s3_string + tune.run(**exp_dict, queue_trials=False, raise_on_failed_trial=False) + + if flags.upload_graphs: + print('Generating experiment graphs and uploading them to leaderboard') + submitter_name, strategy_name = flags.upload_graphs + + # reset ray + ray.shutdown() + if flags.local_mode: + ray.init(local_mode=True) + else: + ray.init() + + # grab checkpoint path + for (dirpath, _, _) in os.walk(os.path.expanduser("~/ray_results")): + if "checkpoint_{}".format(flags.checkpoint_freq) in dirpath \ + and dirpath.split('/')[-3] == flags.exp_title: + checkpoint_path = os.path.dirname(dirpath) + checkpoint_number = -1 + for name in os.listdir(checkpoint_path): + if name.startswith('checkpoint'): + cp = int(name.split('_')[1]) + checkpoint_number = max(checkpoint_number, cp) + + # create dir for graphs output + output_dir = os.path.join(checkpoint_path, 'output_graphs') + if not os.path.exists(output_dir): + os.mkdir(output_dir) + + # run graph generation script + parser = create_parser() + + strategy_name_full = str(strategy_name) + if flags.grid_search: + strategy_name_full += '__' + dirpath.split('/')[-2] + + args = parser.parse_args([ + '-r', checkpoint_path, '-c', str(checkpoint_number), + '--gen_emission', '--use_s3', '--num_cpus', str(flags.num_cpus), + '--output_dir', output_dir, + '--submitter_name', submitter_name, + '--strategy_name', strategy_name_full.replace(',', '_').replace(';', '_') + ]) + generate_graphs(args) def train_h_baselines(env_name, args, multiagent): @@ -292,7 +507,6 @@ def train_stable_baselines(submodule, flags): """Train policies using the PPO algorithm in stable-baselines.""" from stable_baselines.common.vec_env import DummyVecEnv from stable_baselines import PPO2 - flow_params = submodule.flow_params # Path to the saved files exp_tag = flow_params['exp_tag'] diff --git a/flow/algorithms/centralized_PPO.py b/flow/algorithms/centralized_PPO.py new file mode 100644 index 000000000..133d7c8bf --- /dev/null +++ b/flow/algorithms/centralized_PPO.py @@ -0,0 +1,553 @@ +"""An example of customizing PPO to leverage a centralized critic.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import numpy as np + +from ray.rllib.agents.ppo.ppo import PPOTrainer +from flow.algorithms.custom_ppo import CustomPPOTFPolicy, KLCoeffMixin +from ray.rllib.evaluation.postprocessing import compute_advantages, \ + Postprocessing +from ray.rllib.policy.sample_batch import SampleBatch +from ray.rllib.policy.tf_policy import LearningRateSchedule, \ + EntropyCoeffSchedule, ACTION_LOGP +from ray.rllib.models.modelv2 import ModelV2 +from ray.rllib.models.tf.tf_modelv2 import TFModelV2 +from ray.rllib.models.tf.recurrent_tf_modelv2 import RecurrentTFModelV2 +from ray.rllib.utils.annotations import override +from ray.rllib.models.tf.fcnet_v2 import FullyConnectedNetwork +from ray.rllib.utils.explained_variance import explained_variance +from ray.rllib.utils import try_import_tf + +tf = try_import_tf() + +# Frozen logits of the policy that computed the action +BEHAVIOUR_LOGITS = "behaviour_logits" + +CENTRAL_OBS = "central_obs" +OPPONENT_ACTION = "opponent_action" + +parser = argparse.ArgumentParser() +parser.add_argument("--stop", type=int, default=100000) + + +class CentralizedCriticModel(TFModelV2): + """Multi-agent model that implements a centralized VF.""" + + # TODO(@evinitsky) make this work with more than boxes + + def __init__(self, obs_space, action_space, num_outputs, model_config, + name): + super(CentralizedCriticModel, self).__init__( + obs_space, action_space, num_outputs, model_config, name) + # Base of the model + self.model = FullyConnectedNetwork(obs_space, action_space, + num_outputs, model_config, name) + self.register_variables(self.model.variables()) + + # Central VF maps (obs, opp_ops, opp_act) -> vf_pred + self.max_num_agents = model_config['custom_options']['max_num_agents'] + self.obs_space_shape = obs_space.shape[0] + self.obs_space = obs_space + other_obs = tf.keras.layers.Input( + shape=(obs_space.shape[0] * self.max_num_agents,), + name="central_obs") + central_vf_dense = tf.keras.layers.Dense( + model_config['custom_options']['central_vf_size'], + activation=tf.nn.tanh, name="c_vf_dense")(other_obs) + central_vf_out = tf.keras.layers.Dense( + 1, activation=None, name="c_vf_out")(central_vf_dense) + self.central_vf = tf.keras.Model( + inputs=[other_obs], outputs=central_vf_out) + self.register_variables(self.central_vf.variables) + + def forward(self, input_dict, state, seq_lens): + """Run forward inference.""" + return self.model.forward(input_dict, state, seq_lens) + + def central_value_function(self, central_obs): + """Compute the centralized value function.""" + return tf.reshape( + self.central_vf( + [central_obs]), [-1]) + + def value_function(self): + """Compute the normal value function; this is only here to make the code run.""" + return self.model.value_function() # not used + + +# TODO(@evinitsky) support recurrence +class CentralizedCriticModelRNN(RecurrentTFModelV2): + """Example of using the Keras functional API to define a RNN model.""" + + def __init__(self, + obs_space, + action_space, + num_outputs, + model_config, + name, + hiddens_size=64, + cell_size=64): + super(CentralizedCriticModelRNN, self).__init__( + obs_space, action_space, num_outputs, model_config, name) + self.cell_size = cell_size + + # Define input layers + input_layer = tf.keras.layers.Input( + shape=(None, obs_space.shape[0]), name="inputs") + state_in_h = tf.keras.layers.Input(shape=(cell_size,), name="h") + state_in_c = tf.keras.layers.Input(shape=(cell_size,), name="c") + seq_in = tf.keras.layers.Input(shape=(), name="seq_in") + + # Preprocess observation with a hidden layer and send to LSTM cell + dense1 = tf.keras.layers.Dense( + hiddens_size, activation=tf.nn.relu, name="dense1")(input_layer) + lstm_out, state_h, state_c = tf.keras.layers.LSTM( + cell_size, return_sequences=True, return_state=True, name="lstm")( + inputs=dense1, + mask=tf.sequence_mask(seq_in), + initial_state=[state_in_h, state_in_c]) + + # Postprocess LSTM output with another hidden layer and compute values + logits = tf.keras.layers.Dense( + self.num_outputs, + activation=tf.keras.activations.linear, + name="logits")(lstm_out) + values = tf.keras.layers.Dense( + 1, activation=None, name="values")(lstm_out) + + # Create the RNN model + self.model = tf.keras.Model( + inputs=[input_layer, seq_in, state_in_h, state_in_c], + outputs=[logits, values, state_h, state_c]) + self.register_variables(self.model.variables) + self.model.summary() + + # TODO(@evinitsky) add layer sharing to the VF + # Create the centralized VF + # Central VF maps (obs, opp_ops, opp_act) -> vf_pred + self.max_num_agents = model_config.get("max_num_agents", 120) + self.obs_space_shape = obs_space.shape[0] + other_obs = tf.keras.layers.Input( + shape=(obs_space.shape[0] * self.max_num_agents,), + name="all_agent_obs") + central_vf_dense = tf.keras.layers.Dense( + model_config.get("central_vf_size", 64), activation=tf.nn.tanh, + name="c_vf_dense")(other_obs) + central_vf_dense2 = tf.keras.layers.Dense( + model_config.get("central_vf_size", 64), activation=tf.nn.tanh, + name="c_vf_dense")(central_vf_dense) + central_vf_out = tf.keras.layers.Dense( + 1, activation=None, name="c_vf_out")(central_vf_dense2) + self.central_vf = tf.keras.Model( + inputs=[other_obs], outputs=central_vf_out) + self.register_variables(self.central_vf.variables) + + @override(RecurrentTFModelV2) + def forward_rnn(self, inputs, state, seq_lens): + """Forward inference on the RNN.""" + model_out, self._value_out, h, c = self.model( + [inputs, seq_lens] + state) + return model_out, [h, c] + + @override(ModelV2) + def get_initial_state(self): + """Set up the initial RNN state.""" + return [ + np.zeros(self.cell_size, np.float32), + np.zeros(self.cell_size, np.float32), + ] + + def central_value_function(self, central_obs): + """Compute the central value function.""" + return tf.reshape( + self.central_vf( + [central_obs]), [-1]) + + def value_function(self): + """Compute the normal value function; this is only here to make the code run.""" + return tf.reshape(self._value_out, [-1]) # not used + + +class CentralizedValueMixin(object): + """Add methods to evaluate the central value function from the model.""" + + def __init__(self): + # TODO(@evinitsky) clean up naming + self.central_value_function = self.model.central_value_function( + self.get_placeholder(CENTRAL_OBS) + ) + + def compute_central_vf(self, central_obs): + """Run forward inference on the model.""" + feed_dict = { + self.get_placeholder(CENTRAL_OBS): central_obs, + } + return self.get_session().run(self.central_value_function, feed_dict) + + +def centralized_critic_postprocessing(policy, + sample_batch, + other_agent_batches=None, + episode=None): + """Find all other agents that overlapped with you and stack their obs to be passed to the central VF.""" + if policy.loss_initialized(): + assert other_agent_batches is not None + + # time_span = (sample_batch['t'][0], sample_batch['t'][-1]) + # # there's a new problem here, namely that a segment might not be continuous due to the rerouting + # other_agent_timespans = {agent_id: + # (other_agent_batches[agent_id][1]["t"][0], + # other_agent_batches[agent_id][1]["t"][-1]) + # for agent_id in other_agent_batches.keys()} + other_agent_times = {agent_id: other_agent_batches[agent_id][1]["t"] + for agent_id in other_agent_batches.keys()} + agent_time = sample_batch['t'] + # # find agents whose time overlaps with the current agent + rel_agents = {agent_id: other_agent_time for agent_id, other_agent_time + in other_agent_times.items()} + # if len(rel_agents) > 0: + other_obs = { + agent_id: other_agent_batches[agent_id][1]["obs"].copy() + for agent_id in other_agent_batches.keys() + } + padded_agent_obs = { + agent_id: fill_missing( + agent_time, + other_agent_times[agent_id], + other_obs[agent_id] + ) + for agent_id, rel_agent_time in rel_agents.items() + } + # okay, now we need to stack and sort + central_obs_list = [padded_obs for padded_obs in + padded_agent_obs.values()] + try: + central_obs_batch = np.hstack( + (sample_batch["obs"], np.hstack(central_obs_list))) + except Exception as e: + print("Error in centralized PPO: ", e) + # TODO(@ev) this is a bug and needs to be fixed + central_obs_batch = sample_batch["obs"] + max_vf_agents = policy.model.max_num_agents + num_agents = len(rel_agents) + 1 + if num_agents < max_vf_agents: + diff = max_vf_agents - num_agents + zero_pad = np.zeros((central_obs_batch.shape[0], + policy.model.obs_space_shape * diff)) + central_obs_batch = np.hstack((central_obs_batch, + zero_pad)) + elif num_agents > max_vf_agents: + print("Too many agents!") + + # also record the opponent obs and actions in the trajectory + sample_batch[CENTRAL_OBS] = central_obs_batch + + # overwrite default VF prediction with the central VF + sample_batch[SampleBatch.VF_PREDS] = policy.compute_central_vf( + sample_batch[CENTRAL_OBS]) + else: + # policy hasn't initialized yet, use zeros + # TODO(evinitsky) put in the right shape + obs_shape = sample_batch[SampleBatch.CUR_OBS].shape[1] + obs_shape = (1, obs_shape * (policy.model.max_num_agents)) + sample_batch[CENTRAL_OBS] = np.zeros(obs_shape) + # TODO(evinitsky) put in the right shape. Will break if actions aren't 1 + sample_batch[SampleBatch.VF_PREDS] = np.zeros(1, dtype=np.float32) + + # TODO (ak): this was not being used, so commented + # completed = sample_batch["dones"][-1] + + # if not completed and policy.loss_initialized(): + # last_r = 0.0 + # else: + # next_state = [] + # for i in range(policy.num_state_tensors()): + # next_state.append([sample_batch["state_out_{}".format(i)][-1]]) + # last_r = policy.compute_central_vf(sample_batch[CENTRAL_OBS][-1][np.newaxis, ...])[0] + + batch = compute_advantages( + sample_batch, + 0.0, + policy.config["gamma"], + policy.config["lambda"], + use_gae=policy.config["use_gae"]) + return batch + + +def time_overlap(time_span, agent_time): + """Check if agent_time overlaps with time_span.""" + if agent_time[0] <= time_span[1] and agent_time[1] >= time_span[0]: + return True + else: + return False + + +def fill_missing(agent_time, other_agent_time, obs): + """Pad the obs to the appropriate length for agents that don't overlap perfectly in time.""" + # shortcut, the two overlap perfectly + if np.sum(agent_time == other_agent_time) == agent_time.shape[0]: + return obs + new_obs = np.zeros((agent_time.shape[0], obs.shape[1])) + other_agent_time_set = set(other_agent_time) + for i, time in enumerate(agent_time): + if time in other_agent_time_set: + new_obs[i] = obs[np.where(other_agent_time == time)] + return new_obs + + +def overlap_and_pad_agent(time_span, agent_time, obs): + """Take the part of obs that overlaps, pad to length time_span. + + Parameters + ---------- + time_span : tuple + tuple of the first and last time that the agent of interest is in the + system + agent_time : tuple + tuple of the first and last time that the agent whose obs we are + padding is in the system + obs : array_like + observations of the agent whose time is agent_time + """ + assert time_overlap(time_span, agent_time) + print(time_span) + print(agent_time) + # FIXME(ev) some of these conditions can be combined + # no padding needed + if agent_time[0] == time_span[0] and agent_time[1] == time_span[1]: + return obs + # agent enters before time_span starts and exits before time_span end + if agent_time[0] < time_span[0] and agent_time[1] < time_span[1]: + non_overlap_time = time_span[0] - agent_time[0] + missing_time = time_span[1] - agent_time[1] + overlap_obs = obs[non_overlap_time:] + padding = np.zeros((missing_time, obs.shape[1])) + obs_concat = np.concatenate((overlap_obs, padding)) + return obs_concat + # agent enters after time_span starts and exits after time_span ends + elif agent_time[0] > time_span[0] and agent_time[1] > time_span[1]: + non_overlap_time = agent_time[1] - time_span[1] + overlap_obs = obs[:-non_overlap_time] + missing_time = agent_time[0] - time_span[0] + padding = np.zeros((missing_time, obs.shape[1])) + obs_concat = np.concatenate((padding, overlap_obs)) + return obs_concat + # agent time is entirely contained in time_span + elif agent_time[0] >= time_span[0] and agent_time[1] <= time_span[1]: + missing_left = agent_time[0] - time_span[0] + missing_right = time_span[1] - agent_time[1] + obs_concat = obs + if missing_left > 0: + padding = np.zeros((missing_left, obs.shape[1])) + obs_concat = np.concatenate((padding, obs_concat)) + if missing_right > 0: + padding = np.zeros((missing_right, obs.shape[1])) + obs_concat = np.concatenate((obs_concat, padding)) + return obs_concat + # agent time totally contains time_span + elif agent_time[0] <= time_span[0] and agent_time[1] >= time_span[1]: + non_overlap_left = time_span[0] - agent_time[0] + non_overlap_right = agent_time[1] - time_span[1] + overlap_obs = obs + if non_overlap_left > 0: + overlap_obs = overlap_obs[non_overlap_left:] + if non_overlap_right > 0: + overlap_obs = overlap_obs[:-non_overlap_right] + return overlap_obs + + +def loss_with_central_critic(policy, model, dist_class, train_batch): + """Set up the PPO loss but replace the VF loss with the centralized VF loss.""" + CentralizedValueMixin.__init__(policy) + + logits, state = model.from_batch(train_batch) + action_dist = dist_class(logits, model) + + policy.loss_obj = PPOLoss( + policy.action_space, + dist_class, + model, + train_batch[Postprocessing.VALUE_TARGETS], + train_batch[Postprocessing.ADVANTAGES], + train_batch[SampleBatch.ACTIONS], + train_batch[BEHAVIOUR_LOGITS], + train_batch[ACTION_LOGP], + train_batch[SampleBatch.VF_PREDS], + action_dist, + policy.central_value_function, + policy.kl_coeff, + tf.ones_like(train_batch[Postprocessing.ADVANTAGES], dtype=tf.bool), + entropy_coeff=policy.entropy_coeff, + clip_param=policy.config["clip_param"], + vf_clip_param=policy.config["vf_clip_param"], + vf_loss_coeff=policy.config["vf_loss_coeff"], + use_gae=policy.config["use_gae"], + model_config=policy.config["model"]) + + return policy.loss_obj.loss + + +class PPOLoss(object): + """Object containing the PPO loss function.""" + + def __init__(self, + action_space, + dist_class, + model, + value_targets, + advantages, + actions, + prev_logits, + prev_actions_logp, + vf_preds, + curr_action_dist, + value_fn, + cur_kl_coeff, + valid_mask, + entropy_coeff=0, + clip_param=0.1, + vf_clip_param=0.1, + vf_loss_coeff=1.0, + use_gae=True, + model_config=None): + """Construct the loss for Proximal Policy Objective. + + Parameters + ---------- + action_space : TODO + Environment observation space specification. + dist_class : TODO + action distribution class for logits. + value_targets : tf.placeholder + Placeholder for target values; used for GAE. + actions : tf.placeholder + Placeholder for actions taken from previous model evaluation. + advantages : tf.placeholder + Placeholder for calculated advantages from previous model + evaluation. + prev_logits : tf.placeholder + Placeholder for logits output from previous model evaluation. + prev_actions_logp : tf.placeholder + Placeholder for prob output from previous model evaluation. + vf_preds : tf.placeholder + Placeholder for value function output from previous model + evaluation. + curr_action_dist : ActionDistribution + ActionDistribution of the current model. + value_fn : tf.Tensor + Current value function output Tensor. + cur_kl_coeff : tf.Variable + Variable holding the current PPO KL coefficient. + valid_mask : tf.Tensor + A bool mask of valid input elements (#2992). + entropy_coeff : float + Coefficient of the entropy regularizer. + clip_param : float + Clip parameter + vf_clip_param : float + Clip parameter for the value function + vf_loss_coeff : float + Coefficient of the value function loss + use_gae : bool + If true, use the Generalized Advantage Estimator. + model_config : dict, optional + model config for use in specifying action distributions. + """ + + def reduce_mean_valid(t): + return tf.reduce_mean(tf.boolean_mask(t, valid_mask)) + + prev_dist = dist_class(prev_logits, model) + # Make loss functions. + logp_ratio = tf.exp(curr_action_dist.logp(actions) - prev_actions_logp) + action_kl = prev_dist.kl(curr_action_dist) + self.mean_kl = reduce_mean_valid(action_kl) + + curr_entropy = curr_action_dist.entropy() + self.mean_entropy = reduce_mean_valid(curr_entropy) + + surrogate_loss = tf.minimum( + advantages * logp_ratio, + advantages * tf.clip_by_value(logp_ratio, 1 - clip_param, + 1 + clip_param)) + self.mean_policy_loss = reduce_mean_valid(-surrogate_loss) + + if use_gae: + vf_loss1 = tf.square(value_fn - value_targets) + vf_clipped = vf_preds + tf.clip_by_value( + value_fn - vf_preds, -vf_clip_param, vf_clip_param) + vf_loss2 = tf.square(vf_clipped - value_targets) + vf_loss = tf.maximum(vf_loss1, vf_loss2) + self.mean_vf_loss = reduce_mean_valid(vf_loss) + loss = reduce_mean_valid( + -surrogate_loss + + vf_loss_coeff * vf_loss - entropy_coeff * curr_entropy) + else: + self.mean_vf_loss = tf.constant(0.0) + loss = reduce_mean_valid(-surrogate_loss - + entropy_coeff * curr_entropy) + self.loss = loss + + +def new_ppo_surrogate_loss(policy, model, dist_class, train_batch): + """Return the PPO loss with the centralized value function.""" + loss = loss_with_central_critic(policy, model, dist_class, train_batch) + return loss + + +def setup_mixins(policy, obs_space, action_space, config): + """Construct additional classes that add on to PPO.""" + KLCoeffMixin.__init__(policy, config) + + EntropyCoeffSchedule.__init__(policy, config["entropy_coeff"], + config["entropy_coeff_schedule"]) + LearningRateSchedule.__init__(policy, config["lr"], config["lr_schedule"]) + # hack: put in a noop VF so some of the inherited PPO code runs + policy.value_function = tf.zeros( + tf.shape(policy.get_placeholder(SampleBatch.CUR_OBS))[0]) + + +def central_vf_stats(policy, train_batch, grads): + """Report the explained variance of the centralized value function.""" + return { + "vf_explained_var": explained_variance( + train_batch[Postprocessing.VALUE_TARGETS], + policy.central_value_function), + } + + +def kl_and_loss_stats(policy, train_batch): + """Trianing stats to pass to the tensorboard.""" + return { + "cur_kl_coeff": tf.cast(policy.kl_coeff, tf.float64), + "cur_lr": tf.cast(policy.cur_lr, tf.float64), + "total_loss": policy.loss_obj.loss, + "policy_loss": policy.loss_obj.mean_policy_loss, + "vf_loss": policy.loss_obj.mean_vf_loss, + "vf_explained_var": explained_variance( + train_batch[Postprocessing.VALUE_TARGETS], + policy.model.value_function()), + "vf_preds": train_batch[Postprocessing.VALUE_TARGETS], + "kl": policy.loss_obj.mean_kl, + "entropy": policy.loss_obj.mean_entropy, + "entropy_coeff": tf.cast(policy.entropy_coeff, tf.float64), + } + + +CCPPO = CustomPPOTFPolicy.with_updates( + name="CCPPO", + postprocess_fn=centralized_critic_postprocessing, + loss_fn=new_ppo_surrogate_loss, + stats_fn=kl_and_loss_stats, + before_loss_init=setup_mixins, + grad_stats_fn=central_vf_stats, + mixins=[ + LearningRateSchedule, EntropyCoeffSchedule, + CentralizedValueMixin, KLCoeffMixin + ]) + +CCTrainer = PPOTrainer.with_updates(name="CCPPOTrainer", default_policy=CCPPO) diff --git a/flow/algorithms/custom_ppo.py b/flow/algorithms/custom_ppo.py new file mode 100644 index 000000000..47a4459aa --- /dev/null +++ b/flow/algorithms/custom_ppo.py @@ -0,0 +1,344 @@ +"""PPO but without the adaptive KL term that RLlib added.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import logging + +import ray +from ray.rllib.evaluation.postprocessing import compute_advantages, \ + Postprocessing +from ray.rllib.policy.sample_batch import SampleBatch +from ray.rllib.policy.tf_policy import LearningRateSchedule, \ + EntropyCoeffSchedule, ACTION_LOGP +from ray.rllib.policy.tf_policy_template import build_tf_policy +from ray.rllib.utils.explained_variance import explained_variance +from ray.rllib.utils.tf_ops import make_tf_callable +from ray.rllib.utils import try_import_tf + +from ray.rllib.agents.trainer_template import build_trainer +from ray.rllib.agents.ppo.ppo import choose_policy_optimizer, DEFAULT_CONFIG +from ray.rllib.agents.ppo.ppo import warn_about_bad_reward_scales + +tf = try_import_tf() + +logger = logging.getLogger(__name__) + +# Frozen logits of the policy that computed the action +BEHAVIOUR_LOGITS = "behaviour_logits" + + +class PPOLoss(object): + """PPO Loss object.""" + + def __init__(self, + action_space, + dist_class, + model, + value_targets, + advantages, + actions, + prev_logits, + prev_actions_logp, + vf_preds, + curr_action_dist, + value_fn, + cur_kl_coeff, + valid_mask, + entropy_coeff=0, + clip_param=0.1, + vf_clip_param=0.1, + vf_loss_coeff=1.0, + use_gae=True, + model_config=None): + """Construct the loss for Proximal Policy Objective. + + Parameters + ---------- + action_space : TODO + Environment observation space specification. + dist_class : TODO + action distribution class for logits. + value_targets : tf.placeholder + Placeholder for target values; used for GAE. + actions : tf.placeholder + Placeholder for actions taken from previous model evaluation. + advantages : tf.placeholder + Placeholder for calculated advantages from previous model + evaluation. + prev_logits : tf.placeholder + Placeholder for logits output from previous model evaluation. + prev_actions_logp : tf.placeholder + Placeholder for prob output from previous model evaluation. + vf_preds : tf.placeholder + Placeholder for value function output from previous model + evaluation. + curr_action_dist : ActionDistribution + ActionDistribution of the current model. + value_fn : tf.Tensor + Current value function output Tensor. + cur_kl_coeff : tf.Variable + Variable holding the current PPO KL coefficient. + valid_mask : tf.Tensor + A bool mask of valid input elements (#2992). + entropy_coeff : float + Coefficient of the entropy regularizer. + clip_param : float + Clip parameter + vf_clip_param : float + Clip parameter for the value function + vf_loss_coeff : float + Coefficient of the value function loss + use_gae : bool + If true, use the Generalized Advantage Estimator. + model_config : dict, optional + model config for use in specifying action distributions. + """ + + def reduce_mean_valid(t): + return tf.reduce_mean(tf.boolean_mask(t, valid_mask)) + + prev_dist = dist_class(prev_logits, model) + # Make loss functions. + logp_ratio = tf.exp(curr_action_dist.logp(actions) - prev_actions_logp) + action_kl = prev_dist.kl(curr_action_dist) + self.mean_kl = reduce_mean_valid(action_kl) + + curr_entropy = curr_action_dist.entropy() + self.mean_entropy = reduce_mean_valid(curr_entropy) + + surrogate_loss = tf.minimum( + advantages * logp_ratio, + advantages * tf.clip_by_value(logp_ratio, 1 - clip_param, + 1 + clip_param)) + self.mean_policy_loss = reduce_mean_valid(-surrogate_loss) + + if use_gae: + vf_loss1 = tf.square(value_fn - value_targets) + vf_clipped = vf_preds + tf.clip_by_value( + value_fn - vf_preds, -vf_clip_param, vf_clip_param) + vf_loss2 = tf.square(vf_clipped - value_targets) + vf_loss = tf.maximum(vf_loss1, vf_loss2) + self.mean_vf_loss = reduce_mean_valid(vf_loss) + loss = reduce_mean_valid( + -surrogate_loss + + vf_loss_coeff * vf_loss - entropy_coeff * curr_entropy) + else: + self.mean_vf_loss = tf.constant(0.0) + loss = reduce_mean_valid(-surrogate_loss - entropy_coeff * curr_entropy) + self.loss = loss + + +def ppo_surrogate_loss(policy, model, dist_class, train_batch): + """Construct and return the PPO loss.""" + logits, state = model.from_batch(train_batch) + action_dist = dist_class(logits, model) + + if state: + max_seq_len = tf.reduce_max(train_batch["seq_lens"]) + mask = tf.sequence_mask(train_batch["seq_lens"], max_seq_len) + mask = tf.reshape(mask, [-1]) + else: + mask = tf.ones_like( + train_batch[Postprocessing.ADVANTAGES], dtype=tf.bool) + + policy.loss_obj = PPOLoss( + policy.action_space, + dist_class, + model, + train_batch[Postprocessing.VALUE_TARGETS], + train_batch[Postprocessing.ADVANTAGES], + train_batch[SampleBatch.ACTIONS], + train_batch[BEHAVIOUR_LOGITS], + train_batch[ACTION_LOGP], + train_batch[SampleBatch.VF_PREDS], + action_dist, + model.value_function(), + policy.kl_coeff, + mask, + entropy_coeff=policy.entropy_coeff, + clip_param=policy.config["clip_param"], + vf_clip_param=policy.config["vf_clip_param"], + vf_loss_coeff=policy.config["vf_loss_coeff"], + use_gae=policy.config["use_gae"], + model_config=policy.config["model"]) + + return policy.loss_obj.loss + + +def kl_and_loss_stats(policy, train_batch): + """Return statistics for the tensorboard.""" + return { + "cur_kl_coeff": tf.cast(policy.kl_coeff, tf.float64), + "cur_lr": tf.cast(policy.cur_lr, tf.float64), + "total_loss": policy.loss_obj.loss, + "policy_loss": policy.loss_obj.mean_policy_loss, + "vf_loss": policy.loss_obj.mean_vf_loss, + "vf_explained_var": explained_variance( + train_batch[Postprocessing.VALUE_TARGETS], + policy.model.value_function()), + "vf_preds": train_batch[Postprocessing.VALUE_TARGETS], + "kl": policy.loss_obj.mean_kl, + "entropy": policy.loss_obj.mean_entropy, + "entropy_coeff": tf.cast(policy.entropy_coeff, tf.float64), + "advantages": train_batch[Postprocessing.ADVANTAGES], + "rewards": train_batch["rewards"] + } + + +def vf_preds_and_logits_fetches(policy): + """Add value function and logits outputs to experience train_batches.""" + return { + SampleBatch.VF_PREDS: policy.model.value_function(), + BEHAVIOUR_LOGITS: policy.model.last_output(), + } + + +def postprocess_ppo_gae(policy, + sample_batch, + other_agent_batches=None, + episode=None): + """Add the policy logits, VF preds, and advantages to the trajectory.""" + completed = sample_batch["dones"][-1] + if completed: + last_r = 0.0 + else: + next_state = [] + for i in range(policy.num_state_tensors()): + next_state.append([sample_batch["state_out_{}".format(i)][-1]]) + last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1], + sample_batch[SampleBatch.ACTIONS][-1], + sample_batch[SampleBatch.REWARDS][-1], + *next_state) + + batch = compute_advantages( + sample_batch, + last_r, + policy.config["gamma"], + policy.config["lambda"], + use_gae=policy.config["use_gae"]) + return batch + + +def clip_gradients(policy, optimizer, loss): + """If grad_clip is not None, clip the gradients.""" + variables = policy.model.trainable_variables() + if policy.config["grad_clip"] is not None: + grads_and_vars = optimizer.compute_gradients(loss, variables) + grads = [g for (g, v) in grads_and_vars] + policy.grads, _ = tf.clip_by_global_norm(grads, + policy.config["grad_clip"]) + clipped_grads = list(zip(policy.grads, variables)) + return clipped_grads + else: + return optimizer.compute_gradients(loss, variables) + + +class ValueNetworkMixin(object): + """Construct the value function.""" + + def __init__(self, obs_space, action_space, config): + if config["use_gae"]: + + @make_tf_callable(self.get_session()) + def value(ob, prev_action, prev_reward, *state): + model_out, _ = self.model({ + SampleBatch.CUR_OBS: tf.convert_to_tensor([ob]), + SampleBatch.PREV_ACTIONS: tf.convert_to_tensor( + [prev_action]), + SampleBatch.PREV_REWARDS: tf.convert_to_tensor( + [prev_reward]), + "is_training": tf.convert_to_tensor(False), + }, [tf.convert_to_tensor([s]) for s in state], + tf.convert_to_tensor([1])) + return self.model.value_function()[0] + + else: + + @make_tf_callable(self.get_session()) + def value(ob, prev_action, prev_reward, *state): + return tf.constant(0.0) + + self._value = value + + +def setup_config(policy, obs_space, action_space, config): + """Add additional custom options from the config.""" + # auto set the model option for layer sharing + config["model"]["vf_share_layers"] = config["vf_share_layers"] + + +def setup_mixins(policy, obs_space, action_space, config): + """Construct additional classes that add on to PPO.""" + KLCoeffMixin.__init__(policy, config) + ValueNetworkMixin.__init__(policy, obs_space, action_space, config) + EntropyCoeffSchedule.__init__(policy, config["entropy_coeff"], + config["entropy_coeff_schedule"]) + LearningRateSchedule.__init__(policy, config["lr"], config["lr_schedule"]) + + +class KLCoeffMixin(object): + """Update the KL Coefficient. This is intentionally disabled to match the PPO paper better.""" + + def __init__(self, config): + # KL Coefficient + self.kl_coeff_val = config["kl_coeff"] + self.kl_target = config["kl_target"] + self.kl_coeff = tf.get_variable( + initializer=tf.constant_initializer(self.kl_coeff_val), + name="kl_coeff", + shape=(), + trainable=False, + dtype=tf.float32) + + def update_kl(self, blah): + """Disabled to match the PPO paper better.""" + pass + + +CustomPPOTFPolicy = build_tf_policy( + name="CustomPPOTFPolicy", + get_default_config=lambda: ray.rllib.agents.ppo.ppo.DEFAULT_CONFIG, + loss_fn=ppo_surrogate_loss, + stats_fn=kl_and_loss_stats, + extra_action_fetches_fn=vf_preds_and_logits_fetches, + postprocess_fn=postprocess_ppo_gae, + gradients_fn=clip_gradients, + before_init=setup_config, + before_loss_init=setup_mixins, + mixins=[ + LearningRateSchedule, EntropyCoeffSchedule, + ValueNetworkMixin, KLCoeffMixin + ]) + + +def validate_config(config): + """Check that the config is set up properly.""" + if config["entropy_coeff"] < 0: + raise DeprecationWarning("entropy_coeff must be >= 0") + if isinstance(config["entropy_coeff"], int): + config["entropy_coeff"] = float(config["entropy_coeff"]) + if config["batch_mode"] == "truncate_episodes" and not config["use_gae"]: + raise ValueError( + "Episode truncation is not supported without a value " + "function. Consider setting batch_mode=complete_episodes.") + if config["multiagent"]["policies"] and not config["simple_optimizer"]: + logger.info( + "In multi-agent mode, policies will be optimized sequentially " + "by the multi-GPU optimizer. Consider setting " + "simple_optimizer=True if this doesn't work for you.") + if config["simple_optimizer"]: + logger.warning( + "Using the simple minibatch optimizer. This will significantly " + "reduce performance, consider simple_optimizer=False.") + elif tf and tf.executing_eagerly(): + config["simple_optimizer"] = True # multi-gpu not supported + + +CustomPPOTrainer = build_trainer( + name="CustomPPOTrainer", + default_config=DEFAULT_CONFIG, + default_policy=CustomPPOTFPolicy, + make_policy_optimizer=choose_policy_optimizer, + validate_config=validate_config, + after_train_result=warn_about_bad_reward_scales) diff --git a/flow/controllers/__init__.py b/flow/controllers/__init__.py index a61d16980..3f859a83b 100755 --- a/flow/controllers/__init__.py +++ b/flow/controllers/__init__.py @@ -17,13 +17,13 @@ SimCarFollowingController, LACController, GippsController, \ BandoFTLController from flow.controllers.velocity_controllers import FollowerStopper, \ - PISaturation, NonLocalFollowerStopper + PISaturation, NonLocalFollowerStopper, TrajectoryFollower # lane change controllers from flow.controllers.base_lane_changing_controller import \ BaseLaneChangeController from flow.controllers.lane_change_controllers import StaticLaneChanger, \ - SimLaneChangeController + SimLaneChangeController, AILaneChangeController # routing controllers from flow.controllers.base_routing_controller import BaseRouter @@ -37,5 +37,5 @@ "PISaturation", "StaticLaneChanger", "SimLaneChangeController", "ContinuousRouter", "GridRouter", "BayBridgeRouter", "LACController", "GippsController", "NonLocalFollowerStopper", "BandoFTLController", - "I210Router" + "AILaneChangeController", "I210Router", "TrajectoryFollower" ] diff --git a/flow/controllers/base_controller.py b/flow/controllers/base_controller.py index 3c9985360..2045a26ed 100755 --- a/flow/controllers/base_controller.py +++ b/flow/controllers/base_controller.py @@ -2,6 +2,7 @@ from abc import ABCMeta, abstractmethod import numpy as np +import math class BaseController(metaclass=ABCMeta): @@ -66,7 +67,6 @@ def __init__(self, elif isinstance(fail_safe, list) or fail_safe is None: failsafe_list = fail_safe else: - failsafe_list = None raise ValueError("fail_safe should be string or list of strings. Setting fail_safe to None\n") failsafe_map = { @@ -88,14 +88,41 @@ def __init__(self, self.max_accel = car_following_params.controller_params['accel'] # max deaccel should always be a positive self.max_deaccel = abs(car_following_params.controller_params['decel']) + self.max_speed = car_following_params.controller_params['maxSpeed'] self.car_following_params = car_following_params + self._is_highway_i210 = None + @abstractmethod def get_accel(self, env): """Return the acceleration of the controller.""" pass + @abstractmethod + def get_custom_accel(self, this_vel, lead_vel, h): + """Return the custom computed acceleration of the controller. + + This method computes acceleration based on custom state information, + while get_accel() method compute acceleration based on the current state + information that are obtained from the environment. + + Parameters + ---------- + this_vel : float + this vehicle's velocity + lead_vel : float + leading vehicle's velocity + h : float + headway to leading vehicle + + Returns + ------- + float + the custom acceleration of the controller + """ + pass + def get_action(self, env): """Convert the get_accel() acceleration into an action. @@ -117,6 +144,13 @@ def get_action(self, env): float the modified form of the acceleration """ + if self._is_highway_i210 is None: + from flow.networks import I210SubNetwork + from flow.networks import HighwayNetwork + self._is_highway_i210 = \ + isinstance(env.k.network.network, I210SubNetwork) or \ + isinstance(env.k.network.network, HighwayNetwork) + # clear the current stored accels of this vehicle to None env.k.vehicle.update_accel(self.veh_id, None, noise=False, failsafe=False) env.k.vehicle.update_accel(self.veh_id, None, noise=False, failsafe=True) @@ -130,7 +164,8 @@ def get_action(self, env): # this allows the acceleration behavior of vehicles in a junction be # described by sumo instead of an explicit model - if env.k.vehicle.get_edge(self.veh_id)[0] == ":": + if env.k.vehicle.get_edge(self.veh_id)[0] == ":" \ + and not self._is_highway_i210: return None accel = self.get_accel(env) @@ -156,12 +191,17 @@ def get_action(self, env): env.k.vehicle.update_accel(self.veh_id, accel, noise=True, failsafe=False) # run the fail-safes, if requested - for failsafe in self.failsafes: - accel = failsafe(env, accel) + accel = self.compute_failsafe(accel, env) env.k.vehicle.update_accel(self.veh_id, accel, noise=True, failsafe=True) return accel + def compute_failsafe(self, accel, env): + """Take in an acceleration and compute the resultant safe acceleration.""" + for failsafe in self.failsafes: + accel = failsafe(env, accel) + return accel + def get_safe_action_instantaneous(self, env, action): """Perform the "instantaneous" failsafe action. @@ -261,9 +301,16 @@ def safe_velocity(self, env): """Compute a safe velocity for the vehicles. Finds maximum velocity such that if the lead vehicle were to stop - entirely, we can bring the following vehicle to rest at the point at + instantaneously, we can bring the following vehicle to rest at the point at which the headway is zero. + WARNINGS: + 1. We assume the lead vehicle has the same deceleration capabilities as our vehicles + 2. We solve for this value using the discrete time approximation to the dynamics. We assume that the + integration scheme induces positive error in the position, which leads to a slightly more conservative + driving behavior than the continuous time approximation would induce. However, the continuous time + safety rule would not be strictly safe. + Parameters ---------- env : flow.envs.Env @@ -277,17 +324,24 @@ def safe_velocity(self, env): performing the breaking action, and speed limit """ lead_id = env.k.vehicle.get_leader(self.veh_id) + if not lead_id: + return 1000.0 lead_vel = env.k.vehicle.get_speed(lead_id) this_vel = env.k.vehicle.get_speed(self.veh_id) + max_decel = self.max_deaccel + lead_control = env.k.vehicle.get_acc_controller(lead_id) + lead_max_deaccel = lead_control.max_deaccel h = env.k.vehicle.get_headway(self.veh_id) - dv = lead_vel - this_vel + assert (h > 0), print('the headway is less than zero! Seems wrong.') + min_gap = self.car_following_params.controller_params['minGap'] - v_safe = 2 * h / env.sim_step + dv - this_vel * (2 * self.delay) - - # check for speed limit FIXME: this is not called - # this_edge = env.k.vehicle.get_edge(self.veh_id) - # edge_speed_limit = env.k.network.speed_limit(this_edge) + is_ballistic = env.sim_params.use_ballistic + just_inserted = self.veh_id in env.k.vehicle.get_departed_ids() + brake_distance = self.brake_distance(lead_vel, max(max_decel, lead_max_deaccel), + self.delay, is_ballistic, env.sim_step) + v_safe = self.maximum_safe_stop_speed(h + brake_distance - min_gap, this_vel, just_inserted, + is_ballistic, env.sim_step) if this_vel > v_safe: if self.display_warnings: @@ -321,7 +375,10 @@ def get_obey_speed_limit_action(self, env, action): """ # check for speed limit this_edge = env.k.vehicle.get_edge(self.veh_id) - edge_speed_limit = env.k.network.speed_limit(this_edge) + this_lane = env.k.vehicle.get_lane(self.veh_id) + edge_speed_limit = env.k.network.get_max_speed(this_edge, this_lane) + veh_speed_limit = self.max_speed + edge_speed_limit = min(veh_speed_limit, edge_speed_limit) this_vel = env.k.vehicle.get_speed(self.veh_id) sim_step = env.sim_step @@ -379,3 +436,184 @@ def get_feasible_action(self, action): "=====================================".format(self.veh_id)) return action + + def brake_distance(self, speed, max_deaccel, delay, is_ballistic, sim_step): + """Return the distance needed to come to a full stop if braking as hard as possible. + + Parameters + ---------- + speed : float + ego speed + max_deaccel : float + maximum deaccel of the vehicle + delay : float + the delay before an action is executed + is_ballistic : bool + whether the integration stop is ballistic + sim_step : float + size of simulation step + + Returns + ------- + float + the distance required to stop + """ + if is_ballistic: + if speed <= 0: + return 0.0 + else: + return speed * (delay + 0.5 * speed / max_deaccel) + else: + # how much we can reduce the speed in each timestep + speedReduction = max_deaccel * sim_step + # how many steps to get the speed to zero + steps_to_zero = int(speed / speedReduction) + return sim_step * (steps_to_zero * speed - speedReduction * steps_to_zero * (steps_to_zero + 1) / 2) + \ + speed * delay + + def maximum_safe_stop_speed(self, brake_distance, speed, is_inserted, is_ballistic, sim_step): + """Compute the maximum speed that you can travel at and guarantee no collision. + + Parameters + ---------- + brake_distance : float + total distance the vehicle has before it must be at a full stop + speed : float + current vehicle speed + is_inserted : bool + whether the vehicle has just entered the network + is_ballistic : bool + whether the integrator is ballistic + sim_step : float + simulation step size in seconds + + Returns + ------- + v_safe : float + maximum speed that can be travelled at without crashing + """ + if is_ballistic: + v_safe = self.maximum_safe_stop_speed_ballistic(brake_distance, speed, is_inserted, + sim_step) + else: + v_safe = self.maximum_safe_stop_speed_euler(brake_distance, sim_step) + return v_safe + + def maximum_safe_stop_speed_euler(self, brake_distance, sim_step): + """Compute the maximum speed that you can travel at and guarantee no collision for euler integration. + + Parameters + ---------- + brake_distance : float + total distance the vehicle has before it must be at a full stop + sim_step : float + simulation step size in seconds + + Returns + ------- + v_safe : float + maximum speed that can be travelled at without crashing + """ + if brake_distance <= 0: + return 0.0 + + speed_reduction = self.max_deaccel * sim_step + + s = sim_step + t = self.delay + + # h = the distance that would be covered if it were possible to stop + # exactly after gap and decelerate with max_deaccel every simulation step + # h = 0.5 * n * (n-1) * b * s + n * b * t (solve for n) + # n = ((1.0/2.0) - ((t + (pow(((s*s) + (4.0*((s*((2.0*h/b) - t)) + (t*t)))), (1.0/2.0))*sign/2.0))/s)) + sqrt_quantity = math.sqrt( + ((s * s) + (4.0 * ((s * (2.0 * brake_distance / speed_reduction - t)) + (t * t))))) * -0.5 + n = math.floor(.5 - ((t + sqrt_quantity) / s)) + h = 0.5 * n * (n - 1) * speed_reduction * s + n * speed_reduction * t + assert(h <= brake_distance + 1e-6) + # compute the additional speed that must be used during deceleration to fix + # the discrepancy between g and h + r = (brake_distance - h) / (n * s + t) + x = n * speed_reduction + r + assert(x >= 0) + return x + + def maximum_safe_stop_speed_ballistic(self, brake_distance, speed, is_inserted, sim_step): + """Compute the maximum speed that you can travel at and guarantee no collision for ballistic integration. + + Parameters + ---------- + brake_distance : float + total distance the vehicle has before it must be at a full stop + speed : float + current vehicle speed + is_inserted : bool + whether the vehicle has just entered the network + sim_step : float + simulation step size in seconds + + Returns + ------- + v_safe : float + maximum speed that can be travelled at without crashing + """ + # decrease gap slightly (to avoid passing end of lane by values of magnitude ~1e-12, + # when exact stop is required) + new_brake_gap = max(0., brake_distance - 1e-6) + + # (Leo) Note that in contrast to the Euler update, for the ballistic update + # the distance covered in the coming step depends on the current velocity, in general. + # one exception is the situation when the vehicle is just being inserted. + # In that case, it will not cover any distance until the next timestep by convention. + + # We treat the latter case first: + if is_inserted: + # The distance covered with constant insertion speed v0 until time tau is given as + # G1 = tau*v0 + # The distance covered between time tau and the stopping moment at time tau+v0/b is + # G2 = v0^2/(2b), + # where b is an assumed constant deceleration (= myDecel) + # We solve g = G1 + G2 for v0: + btau = self.max_deaccel * self.delay + v0 = -btau + np.sqrt(btau * btau + 2 * self.max_deaccel * new_brake_gap) + return v0 + + # In the usual case during the driving task, the vehicle goes by + # a current speed v0=v, and we seek to determine a safe acceleration a (possibly <0) + # such that starting to break after accelerating with a for the time tau=self.delay + # still allows us to stop in time. + + if self.delay == 0: + tau = sim_step + else: + tau = self.delay + v0 = max(0., speed) + # We first consider the case that a stop has to take place within time tau + if v0 * tau >= 2 * new_brake_gap: + if new_brake_gap == 0: + if v0 > 0.: + # indicate to brake as hard as possible + return -self.max_deaccel * sim_step + else: + # stay stopped + return 0 + # In general we solve g = v0^2/(-2a), where the the rhs is the distance + # covered until stop when breaking with a<0 + a = -v0 * v0 / (2 * new_brake_gap) + return v0 + a * sim_step + + # The last case corresponds to a situation, where the vehicle may go with a positive + # speed v1 = v0 + tau*a after time tau. + # The distance covered until time tau is given as + # G1 = tau*(v0+v1)/2 + # The distance covered between time tau and the stopping moment at time tau+v1/b is + # G2 = v1^2/(2b), + # where b is an assumed constant deceleration (= myDecel) + # We solve g = G1 + G2 for v1>0: + # <=> 0 = v1^2 + b*tau*v1 + b*tau*v0 - 2bg + # => v1 = -b*tau/2 + sqrt( (b*tau)^2/4 + b(2g - tau*v0) ) + + btau2 = self.max_deaccel * tau / 2 + v1 = -btau2 + np.sqrt(btau2 * btau2 + self.max_deaccel * (2 * new_brake_gap - tau * v0)) + a = (v1 - v0) / tau + return v0 + a * sim_step diff --git a/flow/controllers/car_following_models.py b/flow/controllers/car_following_models.py index f5b7399bc..192442980 100755 --- a/flow/controllers/car_following_models.py +++ b/flow/controllers/car_following_models.py @@ -90,6 +90,10 @@ def get_accel(self, env): return self.k_d*(d_l - self.d_des) + self.k_v*(lead_vel - this_vel) + \ self.k_c*(self.v_des - this_vel) + def get_custom_accel(self, this_vel, lead_vel, h): + """See parent class.""" + raise NotImplementedError + class BCMController(BaseController): """Bilateral car-following model controller. @@ -181,6 +185,10 @@ def get_accel(self, env): self.k_v * ((lead_vel - this_vel) - (this_vel - trail_vel)) + \ self.k_c * (self.v_des - this_vel) + def get_custom_accel(self, this_vel, lead_vel, h): + """See parent class.""" + raise NotImplementedError + class LACController(BaseController): """Linear Adaptive Cruise Control. @@ -253,6 +261,10 @@ def get_accel(self, env): return self.a + def get_custom_accel(self, this_vel, lead_vel, h): + """See parent class.""" + raise NotImplementedError + class OVMController(BaseController): """Optimal Vehicle Model controller. @@ -339,6 +351,10 @@ def get_accel(self, env): return self.alpha * (v_h - this_vel) + self.beta * h_dot + def get_custom_accel(self, this_vel, lead_vel, h): + """See parent class.""" + raise NotImplementedError + class LinearOVM(BaseController): """Linear OVM controller. @@ -411,6 +427,10 @@ def get_accel(self, env): return (v_h - this_vel) / self.adaptation + def get_custom_accel(self, this_vel, lead_vel, h): + """See parent class.""" + raise NotImplementedError + class IDMController(BaseController): """Intelligent Driver Model (IDM) controller. @@ -499,6 +519,21 @@ def get_accel(self, env): return self.a * (1 - (v / self.v0)**self.delta - (s_star / h)**2) + def get_custom_accel(self, this_vel, lead_vel, h): + """See parent class.""" + # in order to deal with ZeroDivisionError + if abs(h) < 1e-3: + h = 1e-3 + + if lead_vel is None: # no car ahead + s_star = 0 + else: + s_star = self.s0 + max( + 0, this_vel * self.T + this_vel * (this_vel - lead_vel) / + (2 * np.sqrt(self.a * self.b))) + + return self.a * (1 - (this_vel / self.v0)**self.delta - (s_star / h)**2) + class SimCarFollowingController(BaseController): """Controller whose actions are purely defined by the simulator. @@ -514,6 +549,10 @@ def get_accel(self, env): """See parent class.""" return None + def get_custom_accel(self, this_vel, lead_vel, h): + """See parent class.""" + raise NotImplementedError + class GippsController(BaseController): """Gipps' Model controller. @@ -601,6 +640,10 @@ def get_accel(self, env): return (v_next-v)/env.sim_step + def get_custom_accel(self, this_vel, lead_vel, h): + """See parent class.""" + raise NotImplementedError + class BandoFTLController(BaseController): """Bando follow-the-leader controller. @@ -669,6 +712,7 @@ def __init__(self, def get_accel(self, env): """See parent class.""" + # without generating waves. lead_id = env.k.vehicle.get_leader(self.veh_id) if not lead_id: # no car ahead if self.want_max_accel: @@ -685,3 +729,7 @@ def accel_func(self, v, v_l, s): s_dot = v_l - v u = self.alpha * (v_h - v) + self.beta * s_dot/(s**2) return u + + def get_custom_accel(self, this_vel, lead_vel, h): + """See parent class.""" + raise NotImplementedError diff --git a/flow/controllers/lane_change_controllers.py b/flow/controllers/lane_change_controllers.py index 7f7a16ff3..cc268bd3b 100755 --- a/flow/controllers/lane_change_controllers.py +++ b/flow/controllers/lane_change_controllers.py @@ -24,3 +24,167 @@ class StaticLaneChanger(BaseLaneChangeController): def get_lane_change_action(self, env): """See parent class.""" return 0 + + +class AILaneChangeController(BaseLaneChangeController): + """A lane-changing controller based on acceleration incentive model. + + Usage + ----- + See base class for usage example. + + Attributes + ---------- + veh_id : str + Vehicle ID for SUMO/Aimsun identification + lane_change_params : flow.core.param.SumoLaneChangeParams + see parent class + left_delta : float + used for the incentive criterion for left lane change (default: 2.6) + right_delta : float + used for the incentive criterion for right lane change (default: 2.7) + left_beta : float + used for the incentive criterion for left lane change (default: 2.6) + right_beta : float + used for the incentive criterion for right lane change (default: 2.7) + """ + + def __init__(self, + veh_id, + lane_change_params=None, + left_delta=2.6, + right_delta=2.7, + left_beta=2.6, + right_beta=2.7): + """Instantiate an AI lane-change controller.""" + BaseLaneChangeController.__init__( + self, + veh_id, + lane_change_params, + ) + + self.veh_id = veh_id + self.left_delta = left_delta + self.right_delta = right_delta + self.left_beta = left_beta + self.right_beta = right_beta + + def get_lane_change_action(self, env): + """See parent class.""" + # acceleration if the ego vehicle remains in current lane. + ego_accel_controller = env.k.vehicle.get_acc_controller(self.veh_id) + acc_in_present_lane = ego_accel_controller.get_accel(env) + + # get ego vehicle lane number, and velocity + ego_lane = env.k.vehicle.get_lane(self.veh_id) + ego_vel = env.k.vehicle.get_speed(self.veh_id) + + # get lane leaders, followers, headways, and tailways + lane_leaders = env.k.vehicle.get_lane_leaders(self.veh_id) + lane_followers = env.k.vehicle.get_lane_followers(self.veh_id) + lane_headways = env.k.vehicle.get_lane_headways(self.veh_id) + lane_tailways = env.k.vehicle.get_lane_tailways(self.veh_id) + + # determine left and right lane number + this_edge = env.k.vehicle.get_edge(self.veh_id) + num_lanes = env.k.network.num_lanes(this_edge) + l_lane = ego_lane - 1 if ego_lane > 0 else None + r_lane = ego_lane + 1 if ego_lane < num_lanes - 1 else None + + # compute ego and new follower accelerations if moving to left lane + if l_lane is not None: + # get left leader and follower vehicle ID + l_l = lane_leaders[l_lane] + l_f = lane_followers[l_lane] + + # ego acceleration if the ego vehicle is in the lane to the left + if l_l not in ['', None]: + # left leader velocity and headway + l_l_vel = env.k.vehicle.get_speed(l_l) + l_l_headway = lane_headways[l_lane] + acc_in_left_lane = ego_accel_controller.get_custom_accel( + this_vel=ego_vel, + lead_vel=l_l_vel, + h=l_l_headway) + else: # if left lane exists but left leader does not exist + # in this case we assign None to the leader velocity and + # large number to headway + l_l_vel = None + l_l_headway = 1000 + acc_in_left_lane = ego_accel_controller.get_custom_accel( + this_vel=ego_vel, + lead_vel=l_l_vel, + h=l_l_headway) + + # follower acceleration if the ego vehicle is in the left lane + if l_f not in ['', None]: + # left follower velocity and headway + l_f_vel = env.k.vehicle.get_speed(l_f) + l_f_tailway = lane_tailways[l_lane] + l_f_accel_controller = env.k.vehicle.get_acc_controller(l_f) + left_lane_follower_acc = l_f_accel_controller.get_custom_accel( + this_vel=l_f_vel, + lead_vel=ego_vel, + h=l_f_tailway) + else: # if left lane exists but left follower does not exist + # in this case we assign maximum acceleration + left_lane_follower_acc = ego_accel_controller.max_accel + else: + acc_in_left_lane = None + left_lane_follower_acc = None + + # compute ego and new follower accelerations if moving to right lane + if r_lane is not None: + # get right leader and follower vehicle ID + r_l = lane_leaders[r_lane] + r_f = lane_followers[r_lane] + + # ego acceleration if the ego vehicle is in the lane to the right + if r_l not in ['', None]: + # right leader velocity and headway + r_l_vel = env.k.vehicle.get_speed(r_l) + r_l_headway = lane_headways[r_lane] + acc_in_right_lane = ego_accel_controller.get_custom_accel( + this_vel=ego_vel, + lead_vel=r_l_vel, + h=r_l_headway) + else: # if right lane exists but right leader does not exist + # in this case we assign None to the leader velocity and + # large number to headway + r_l_vel = None + r_l_headway = 1000 + acc_in_right_lane = ego_accel_controller.get_custom_accel( + this_vel=ego_vel, + lead_vel=r_l_vel, + h=r_l_headway) + + # follower acceleration if the ego vehicle is in the right lane + if r_f not in ['', None]: + # right follower velocity and headway + r_f_vel = env.k.vehicle.get_speed(r_f) + r_f_headway = lane_tailways[r_lane] + r_f_accel_controller = env.k.vehicle.get_acc_controller(r_f) + right_lane_follower_acc = r_f_accel_controller.get_custom_accel( + this_vel=r_f_vel, + lead_vel=ego_vel, + h=r_f_headway) + else: # if right lane exists but right follower does not exist + # assign maximum acceleration + right_lane_follower_acc = ego_accel_controller.max_accel + else: + acc_in_right_lane = None + right_lane_follower_acc = None + + # determine lane change action + if l_lane is not None and acc_in_left_lane >= - self.left_beta and \ + left_lane_follower_acc >= -self.left_beta and \ + acc_in_left_lane >= acc_in_present_lane + self.left_delta: + action = 1 + elif r_lane is not None and acc_in_right_lane >= - self.right_beta and \ + right_lane_follower_acc >= -self.right_beta and \ + acc_in_right_lane >= acc_in_present_lane + self.right_delta: + action = -1 + else: + action = 0 + + return action diff --git a/flow/controllers/rlcontroller.py b/flow/controllers/rlcontroller.py index 973de8fc9..c4100a373 100755 --- a/flow/controllers/rlcontroller.py +++ b/flow/controllers/rlcontroller.py @@ -31,13 +31,19 @@ class RLController(BaseController): >>> rl_ids = env.k.vehicle.get_rl_ids() """ - def __init__(self, veh_id, car_following_params): + def __init__(self, veh_id, car_following_params, fail_safe=None): """Instantiate an RL Controller.""" BaseController.__init__( self, veh_id, - car_following_params) + car_following_params, + fail_safe=fail_safe, + ) def get_accel(self, env): """Pass, as this is never called; required to override abstractmethod.""" pass + + def get_custom_accel(self, this_vel, lead_vel, h): + """See parent class.""" + raise NotImplementedError diff --git a/flow/controllers/velocity_controllers.py b/flow/controllers/velocity_controllers.py index 2e4b7c22a..ac351ad95 100644 --- a/flow/controllers/velocity_controllers.py +++ b/flow/controllers/velocity_controllers.py @@ -20,17 +20,27 @@ class FollowerStopper(BaseController): unique vehicle identifier v_des : float, optional desired speed of the vehicles (m/s) + no_control_edges : [str] + list of edges that we should not apply control on """ def __init__(self, veh_id, car_following_params, v_des=15, - danger_edges=None): + fail_safe=None, + danger_edges=None, + control_length=None, + no_control_edges=None): """Instantiate FollowerStopper.""" - BaseController.__init__( - self, veh_id, car_following_params, delay=1.0, - fail_safe='safe_velocity') + if fail_safe: + BaseController.__init__( + self, veh_id, car_following_params, delay=0.0, + fail_safe=fail_safe) + else: + BaseController.__init__( + self, veh_id, car_following_params, delay=0.0, + fail_safe='safe_velocity') # desired speed of the vehicle self.v_des = v_des @@ -45,7 +55,10 @@ def __init__(self, self.d_1 = 1.5 self.d_2 = 1.0 self.d_3 = 0.5 + self.danger_edges = danger_edges if danger_edges else {} + self.control_length = control_length + self.no_control_edges = no_control_edges def find_intersection_dist(self, env): """Find distance to intersection. @@ -107,13 +120,17 @@ def get_accel(self, env): if edge == "": return None - if self.find_intersection_dist(env) <= 10 and \ - env.k.vehicle.get_edge(self.veh_id) in self.danger_edges or \ + if (self.find_intersection_dist(env) <= 10 and + env.k.vehicle.get_edge(self.veh_id) in self.danger_edges) or \ env.k.vehicle.get_edge(self.veh_id)[0] == ":": return None else: # compute the acceleration from the desired velocity - return (v_cmd - this_vel) / env.sim_step + return np.clip((v_cmd - this_vel) / env.sim_step, -np.abs(self.max_deaccel), self.max_accel) + + def get_custom_accel(self, this_vel, lead_vel, h): + """See parent class.""" + raise NotImplementedError class NonLocalFollowerStopper(FollowerStopper): @@ -154,11 +171,6 @@ def get_accel(self, env): if edge == "": return None - - if self.find_intersection_dist(env) <= 10 and \ - env.k.vehicle.get_edge(self.veh_id) in self.danger_edges or \ - env.k.vehicle.get_edge(self.veh_id)[0] == ":": - return None else: # compute the acceleration from the desired velocity return (v_cmd - this_vel) / env.sim_step @@ -184,7 +196,7 @@ class PISaturation(BaseController): def __init__(self, veh_id, car_following_params): """Instantiate PISaturation.""" - BaseController.__init__(self, veh_id, car_following_params, delay=1.0) + BaseController.__init__(self, veh_id, car_following_params, delay=0.0) # maximum achievable acceleration by the vehicle self.max_accel = car_following_params.controller_params['accel'] @@ -238,3 +250,55 @@ def get_accel(self, env): accel = (self.v_cmd - this_vel) / env.sim_step return min(accel, self.max_accel) + + def get_custom_accel(self, this_vel, lead_vel, h): + """See parent class.""" + raise NotImplementedError + + +class TrajectoryFollower(BaseController): + """Follow a set trajectory. + + Usage + ----- + See base class for example. + + Parameters + ---------- + veh_id : str + unique vehicle identifier + car_following_params : flow.core.params.SumoCarFollowingParams + object defining sumo-specific car-following parameters + func : function f that defines trajectory as speed = f(t), where t is env.time_counter + """ + + def __init__(self, veh_id, car_following_params, func): + """Instantiate TrajectoryFollower.""" + BaseController.__init__(self, veh_id, car_following_params, delay=0.0, fail_safe=['instantaneous', + 'feasible_accel', + 'obey_speed_limit']) + # maximum achievable acceleration by the vehicle + self.max_accel = car_following_params.controller_params['accel'] + + # history used to determine AV desired velocity + self.v_history = [] + self.v_target = 0 + self.v_cmd = 0 + self.speed_func = func + + def get_accel(self, env): + """See parent class.""" + this_vel = env.k.vehicle.get_speed(self.veh_id) + # update the AV's velocity history + self.v_history.append(this_vel) + # compute desired velocity + self.v_cmd = max(self.speed_func(env.time_counter), 0) + # compute the acceleration + accel = (self.v_cmd - this_vel) / env.sim_step + this_edge = env.k.vehicle.get_edge(self.veh_id) + edge_speed_limit = env.k.network.speed_limit(this_edge) + if (self.v_cmd > (edge_speed_limit - 5) and this_vel == self.v_history[-2]) or \ + (self.v_cmd < 0 and this_vel == 0): + accel = 0 + + return min(accel, self.max_accel) diff --git a/flow/core/experiment.py b/flow/core/experiment.py index d97f96582..876c99da9 100755 --- a/flow/core/experiment.py +++ b/flow/core/experiment.py @@ -1,9 +1,21 @@ """Contains an experiment class for running simulations.""" from flow.utils.registry import make_create_env +from flow.core.util import ensure_dir +from flow.data_pipeline.data_pipeline import upload_to_s3 +from flow.data_pipeline.data_pipeline import get_configuration +from flow.data_pipeline.data_pipeline import generate_trajectory_table +from flow.data_pipeline.data_pipeline import write_dict_to_csv +from flow.data_pipeline.leaderboard_utils import network_name_translate +from flow.core.rewards import veh_energy_consumption, instantaneous_mpg +from flow.visualize.time_space_diagram import tsd_main +from collections import defaultdict +from datetime import timezone from datetime import datetime import logging import time import numpy as np +import uuid +import os class Experiment: @@ -57,7 +69,10 @@ class can generate csv files from emission files produced by sumo. These the environment object the simulator will run """ - def __init__(self, flow_params, custom_callables=None): + def __init__(self, + flow_params, + custom_callables=None, + register_with_ray=False): """Instantiate the Experiment class. Parameters @@ -69,21 +84,39 @@ def __init__(self, flow_params, custom_callables=None): want to extract from the environment. The lambda will be called at each step to extract information from the env and it will be stored in a dict keyed by the str. + register_with_ray : bool + whether the environment is meant to be registered with ray. If set + to True, the environment is including as part of `self.env` + separately (i.e. not here). """ self.custom_callables = custom_callables or {} # Get the env name and a creator for the environment. - create_env, _ = make_create_env(flow_params) + create_env, env_name = make_create_env(flow_params) - # Create the environment. - self.env = create_env() - - logging.info(" Starting experiment {} at {}".format( - self.env.network.name, str(datetime.utcnow()))) - - logging.info("Initializing environment.") + # record env_name and create_env, need it to register for ray + self.env_name = env_name + self.create_env = create_env - def run(self, num_runs, rl_actions=None, convert_to_csv=False): + # Create the environment. + if not register_with_ray: + self.env = create_env() + + logging.info(" Starting experiment {} at {}".format( + self.env.network.name, str(datetime.utcnow()))) + + logging.info("Initializing environment.") + + def run(self, + num_runs, + rl_actions=None, + convert_to_csv=False, + to_aws=None, + is_baseline=False, + multiagent=False, + rets=None, + policy_map_fn=None, + supplied_metadata=None): """Run the given network for a set number of runs. Parameters @@ -96,6 +129,20 @@ def run(self, num_runs, rl_actions=None, convert_to_csv=False): convert_to_csv : bool Specifies whether to convert the emission file created by sumo into a csv file + to_aws: str + Specifies the S3 partition you want to store the output file, + will be used to later for query. If NONE, won't upload output + to S3. + is_baseline: bool + Specifies whether this is a baseline run. + multiagent : bool + whether the policy is multi-agent + rets : dict + a dictionary to store the rewards for multiagent simulation + policy_map_fn : function + a mapping from each agent to their respective policy + supplied_metadata: dict (str: list) + metadata provided by the caller Returns ------- @@ -117,12 +164,26 @@ def run(self, num_runs, rl_actions=None, convert_to_csv=False): 'output should be generated. If you do not wish to generate ' 'emissions, set the convert_to_csv parameter to False.') + # Make sure the emission path directory exists, and if not, create it. + if self.env.sim_params.emission_path is not None: + ensure_dir(self.env.sim_params.emission_path) + # used to store info_dict = { - "returns": [], "velocities": [], "outflows": [], + "avg_trip_energy": [], + "avg_trip_time": [], + "total_completed_trips": [], + "inst_mpg": [], + "avg_accel_human": [], + "avg_headway": [], } + if not multiagent: + info_dict["returns"] = [] + all_trip_energy_distribution = defaultdict(lambda: []) + all_trip_time_distribution = defaultdict(lambda: []) + info_dict.update({ key: [] for key in self.custom_callables.keys() }) @@ -135,11 +196,64 @@ def rl_actions(*_): t = time.time() times = [] + if convert_to_csv and self.env.simulator == "traci": + # data pipeline + source_id = 'flow_{}'.format(uuid.uuid4().hex) + metadata = defaultdict(lambda: []) + + # collect current time + cur_datetime = datetime.now(timezone.utc) + cur_date = cur_datetime.date().isoformat() + cur_time = cur_datetime.time().isoformat() + + if to_aws: + # collecting information for metadata table + metadata['source_id'].append(source_id) + metadata['submission_time'].append(cur_time) + metadata['network'].append( + network_name_translate(self.env.network.name.split('_20')[0])) + metadata['is_baseline'].append(str(is_baseline)) + if supplied_metadata is not None \ + and 'submitter_name' in supplied_metadata and 'strategy' in supplied_metadata: + name = supplied_metadata.get('submitter_name') + strategy = supplied_metadata.get('strategy') + else: + name, strategy = get_configuration() + metadata['submitter_name'].append(name) + metadata['strategy'].append(strategy) + if supplied_metadata is not None: + metadata['version'].append(supplied_metadata.get('version')) + metadata['on_ramp'].append(supplied_metadata.get('on_ramp')) + metadata['penetration_rate'].append(supplied_metadata.get('penetration_rate')) + metadata['road_grade'].append(supplied_metadata.get('road_grade')) + + # emission-specific parameters + dir_path = self.env.sim_params.emission_path + trajectory_table_path = os.path.join( + dir_path, '{}.csv'.format(source_id)) + metadata_table_path = os.path.join( + dir_path, '{}_METADATA.csv'.format(source_id)) + else: + source_id = None + trajectory_table_path = None + metadata_table_path = None + metadata = None + cur_date = None + + emission_files = [] for i in range(num_runs): - ret = 0 + if rets and multiagent: + ret = {key: [0] for key in rets.keys()} + else: + ret = 0 vel = [] + per_vehicle_energy_trace = defaultdict(lambda: []) + completed_veh_types = {} + completed_vehicle_avg_energy = {} + completed_vehicle_travel_time = {} custom_vals = {key: [] for key in self.custom_callables.keys()} state = self.env.reset() + initial_vehicles = set(self.env.k.vehicle.get_ids()) for j in range(num_steps): t0 = time.time() state, reward, done, _ = self.env.step(rl_actions(state)) @@ -149,29 +263,94 @@ def rl_actions(*_): # Compute the velocity speeds and cumulative returns. veh_ids = self.env.k.vehicle.get_ids() vel.append(np.mean(self.env.k.vehicle.get_speed(veh_ids))) - ret += reward + if rets and multiagent: + for actor, rew in reward.items(): + ret[policy_map_fn(actor)][0] += rew + elif not multiagent: + ret += reward # Compute the results for the custom callables. for (key, lambda_func) in self.custom_callables.items(): custom_vals[key].append(lambda_func(self.env)) - if done: + # Compute the results for energy metrics + for past_veh_id in per_vehicle_energy_trace.keys(): + if past_veh_id not in veh_ids and past_veh_id not in completed_vehicle_avg_energy: + all_trip_energy_distribution[completed_veh_types[past_veh_id]].append( + np.sum(per_vehicle_energy_trace[past_veh_id])) + all_trip_time_distribution[completed_veh_types[past_veh_id]].append( + len(per_vehicle_energy_trace[past_veh_id])) + completed_vehicle_avg_energy[past_veh_id] = np.sum(per_vehicle_energy_trace[past_veh_id]) + completed_vehicle_travel_time[past_veh_id] = len(per_vehicle_energy_trace[past_veh_id]) + + # Update the stored energy metrics calculation results + for veh_id in veh_ids: + if veh_id not in initial_vehicles: + if veh_id not in per_vehicle_energy_trace: + # we have to skip the first step's energy calculation + per_vehicle_energy_trace[veh_id].append(0) + completed_veh_types[veh_id] = self.env.k.vehicle.get_type(veh_id) + else: + per_vehicle_energy_trace[veh_id].append(-1 * veh_energy_consumption(self.env, veh_id)) + + if multiagent and done['__all__']: break + if type(done) is dict and done['__all__'] or done is True: + break + + if rets and multiagent: + for key in rets.keys(): + rets[key].append(ret[key]) # Store the information from the run in info_dict. + if hasattr(self.env, 'no_control_edges'): + veh_ids = [ + veh_id for veh_id in self.env.k.vehicle.get_ids() + if self.env.k.vehicle.get_speed(veh_id) >= 0 + and self.env.k.vehicle.get_edge(veh_id) not in self.env.no_control_edges + ] + # rl_ids = [ + # veh_id for veh_id in self.env.k.vehicle.get_rl_ids() + # if self.env.k.vehicle.get_speed(veh_id) >= 0 + # and self.env.k.vehicle.get_edge(veh_id) not in self.env.no_control_edges + # ] + else: + veh_ids = [veh_id for veh_id in self.env.k.vehicle.get_ids() + if self.env.k.vehicle.get_speed(veh_id) >= 0] + # rl_ids = [veh_id for veh_id in self.env.k.vehicle.get_rl_ids() + # if self.env.k.vehicle.get_speed(veh_id) >= 0] + outflow = self.env.k.vehicle.get_outflow_rate(int(500)) - info_dict["returns"].append(ret) + if not multiagent: + info_dict["returns"].append(ret) info_dict["velocities"].append(np.mean(vel)) info_dict["outflows"].append(outflow) + info_dict["avg_trip_energy"].append(np.mean(list(completed_vehicle_avg_energy.values()))) + info_dict["avg_trip_time"].append(np.mean(list(completed_vehicle_travel_time.values()))) + info_dict["total_completed_trips"].append(len(list(completed_vehicle_avg_energy.values()))) + info_dict["lane_change_count"] = self.env.k.vehicle.lane_change_count + info_dict["inst_mpg"].append(instantaneous_mpg(self.env, veh_ids, gain=1.0)) + info_dict["avg_accel_human"].append(np.nan_to_num(np.mean( + [np.abs((self.env.k.vehicle.get_speed(veh_id) - + self.env.k.vehicle.get_previous_speed(veh_id))/self.env.sim_step) for + veh_id in veh_ids if veh_id in self.env.k.vehicle.previous_speeds.keys()] + ))) + info_dict["avg_headway"].append(np.mean(self.env.k.vehicle.get_headway(veh_ids))) + for key in custom_vals.keys(): info_dict[key].append(np.mean(custom_vals[key])) - print("Round {0}, return: {1}".format(i, ret)) + if rets and multiagent: + for agent_id, rew in rets.items(): + print('Round {}, Return: {} for agent {}'.format( + i, ret, agent_id)) + elif not multiagent: + print('Round {}, Return: {}'.format(i, ret)) # Save emission data at the end of every rollout. This is skipped # by the internal method if no emission path was specified. if self.env.simulator == "traci": - self.env.k.simulation.save_emission(run_id=i) + emission_files.append(self.env.k.simulation.save_emission(run_id=i)) # Print the averages/std for all variables in the info_dict. for key in info_dict.keys(): @@ -182,4 +361,39 @@ def rl_actions(*_): print("steps/second:", np.mean(times)) self.env.terminate() + if to_aws: + write_dict_to_csv(metadata_table_path, metadata, True) + generate_trajectory_table(emission_files, trajectory_table_path, source_id) + tsd_main( + emission_files[0], + { + 'network': self.env.network.__class__, + 'env': self.env.env_params, + 'sim': self.env.sim_params + }, + min_speed=0, + max_speed=10 + ) + upload_to_s3( + 'circles.data.pipeline', + 'metadata_table/date={0}/partition_name={1}_METADATA/' + '{1}_METADATA.csv'.format(cur_date, source_id), + metadata_table_path + ) + upload_to_s3( + 'circles.data.pipeline', + 'fact_vehicle_trace/date={0}/partition_name={1}/' + '{1}.csv'.format(cur_date, source_id), + trajectory_table_path, + {'network': metadata['network'][0], + 'is_baseline': metadata['is_baseline'][0]} + ) + upload_to_s3( + 'circles.data.pipeline', + 'time_space_diagram/date={0}/partition_name={1}/' + '{1}.png'.format(cur_date, source_id), + emission_files[0].replace('csv', 'png') + ) + os.remove(trajectory_table_path) + return info_dict diff --git a/flow/core/kernel/network/traci.py b/flow/core/kernel/network/traci.py index c9ac80772..abf8fdcf6 100644 --- a/flow/core/kernel/network/traci.py +++ b/flow/core/kernel/network/traci.py @@ -83,6 +83,7 @@ def __init__(self, master_kernel, sim_params): self.__non_internal_length = None # total length of non-internal edges self.rts = None self.cfg = None + self._speed_limit = {} def generate_network(self, network): """See parent class. @@ -211,8 +212,12 @@ def generate_network(self, network): self.cfg = self.cfg_path + cfg_name def update(self, reset): - """Perform no action of value (networks are static).""" - pass + """See parent class. + + This methods clears the speed limit dict, which may have been updated + dynamically. The dict is then filled by the get_max_speed method. + """ + self._speed_limit.clear() def close(self): """Close the network class. @@ -560,7 +565,7 @@ def generate_net_from_osm(self, net_params): netfn = "%s.net.xml" % self.name # generate the network file with sumo - net_cmd = "netconvert --osm-files {0} --output-file {1}".\ + net_cmd = "netconvert --osm-files {0} --output-file {1}". \ format(osm_path, self.cfg_path + netfn) # this handles removing all roads in the network that cannot be ridden @@ -935,3 +940,33 @@ def _import_edges_from_net(self, net_params): connection_data = {'next': next_conn_data, 'prev': prev_conn_data} return net_data, connection_data + + def set_max_speed(self, edge_id, speed): + """Set the max speed of a network edge. + + Parameters + ---------- + edge_id : str + ID of a network edge + speed : float + Max speed of the network edge + + """ + self.kernel_api.edge.setMaxSpeed(edge_id, speed) + + def get_max_speed(self, edge_id, lane_id): + """Get the max speed of a network lane. + + Parameters + ---------- + edge_id : str + ID of a network edge + lane_id : str + Index of a lane + """ + edge = edge_id + '_' + str(lane_id) + + if edge not in self._speed_limit.keys(): + self._speed_limit[edge] = self.kernel_api.lane.getMaxSpeed(edge) + + return self._speed_limit[edge] diff --git a/flow/core/kernel/simulation/traci.py b/flow/core/kernel/simulation/traci.py index 387f7b03a..d476c7298 100644 --- a/flow/core/kernel/simulation/traci.py +++ b/flow/core/kernel/simulation/traci.py @@ -192,6 +192,11 @@ def start_simulation(self, network, sim_params): "--step-length", str(sim_params.sim_step) ] + # disable all collisions and teleporting in the simulation. + if sim_params.disable_collisions: + sumo_call.extend(["--collision.mingap-factor", str(0), + "--collision.action", str("none")]) + # use a ballistic integration step (if request) if sim_params.use_ballistic: sumo_call.append("--step-method.ballistic") @@ -226,6 +231,18 @@ def start_simulation(self, network, sim_params): sumo_call.append("--collision.check-junctions") sumo_call.append("true") + # save an initial state, if requested + if sim_params.save_state_time is not None: + sumo_call.append("--save-state.times") + sumo_call.append(str(sim_params.save_state_time)) + sumo_call.append("--save-state.files") + sumo_call.append(sim_params.save_state_file) + + # add the initial state of vehicles in the network, if provided + if sim_params.load_state is not None: + sumo_call.append("--load-state") + sumo_call.append(sim_params.load_state) + logging.info(" Starting SUMO on port " + str(port)) logging.debug(" Cfg file: " + str(network.cfg)) if sim_params.num_clients > 1: @@ -269,14 +286,19 @@ def teardown_sumo(self): def save_emission(self, run_id=0): """Save any collected emission data to a csv file. - If not data was collected, nothing happens. Moreover, any internally - stored data by this class is clear whenever data is stored. + If no data was collected, nothing happens. Moreover, any internally + stored data by this class is cleared whenever data is stored. Parameters ---------- run_id : int the rollout number, appended to the name of the emission file. Used to store emission files from multiple rollouts run sequentially. + + Returns + ------- + emission_file_path: str + the relative path of the emission file """ # If there is no stored data, ignore this operation. This is to ensure # that data isn't deleted if the operation is called twice. @@ -294,6 +316,8 @@ def save_emission(self, run_id=0): "speed", "headway", "leader_id", + "follower_id", + "leader_rel_speed", "target_accel_with_noise_with_failsafe", "target_accel_no_noise_no_failsafe", "target_accel_with_noise_no_failsafe", @@ -304,8 +328,6 @@ def save_emission(self, run_id=0): "lane_number", "distance", "relative_position", - "follower_id", - "leader_rel_speed", ] # Update the stored data to push to the csv file. @@ -319,8 +341,9 @@ def save_emission(self, run_id=0): for key in stored_ids: final_data[key].append(self.stored_data[veh_id][t][key]) - with open(os.path.join(self.emission_path, name), "w") as f: - print(os.path.join(self.emission_path, name), self.emission_path) + emission_file_path = os.path.join(self.emission_path, name) + with open(emission_file_path, "w") as f: + print(emission_file_path, self.emission_path) writer = csv.writer(f, delimiter=',') writer.writerow(final_data.keys()) writer.writerows(zip(*final_data.values())) @@ -328,3 +351,5 @@ def save_emission(self, run_id=0): # Clear all memory from the stored data. This is useful if this # function is called in between resets. self.stored_data.clear() + + return emission_file_path diff --git a/flow/core/kernel/vehicle/aimsun.py b/flow/core/kernel/vehicle/aimsun.py index 16c94558a..6e49b7987 100644 --- a/flow/core/kernel/vehicle/aimsun.py +++ b/flow/core/kernel/vehicle/aimsun.py @@ -664,10 +664,6 @@ def get_num_not_departed(self): """See parent class.""" raise NotImplementedError - def get_fuel_consumption(self): - """See parent class.""" - raise NotImplementedError - def get_type(self, veh_id): """See parent class.""" if isinstance(veh_id, (list, np.ndarray)): diff --git a/flow/core/kernel/vehicle/base.py b/flow/core/kernel/vehicle/base.py index a433b8924..e7cb0f29a 100644 --- a/flow/core/kernel/vehicle/base.py +++ b/flow/core/kernel/vehicle/base.py @@ -128,10 +128,13 @@ def remove(self, veh_id): pass @abstractmethod - def apply_acceleration(self, veh_id, acc, smooth=True): + def apply_acceleration(self, veh_id, acc, smooth_duration=0): """Apply the acceleration requested by a vehicle in the simulator. - In SUMO, this function applies slowDown method which applies smoothing. + In SUMO, this function applies setSpeed for smooth_duration=0, otherwise + the slowDown method applies acceleration smoothly over the smooth_duration + time (in seconds). For more information, see: + https://sumo.dlr.de/pydoc/traci._vehicle.html#VehicleDomain-slowDown Parameters ---------- @@ -139,8 +142,8 @@ def apply_acceleration(self, veh_id, acc, smooth=True): list of vehicle identifiers acc : float or array_like requested accelerations from the vehicles - smooth : bool - whether to apply acceleration smoothly or not, default: True + smooth_duration : float + duration in seconds over which acceleration should be smoothly applied, default: 0 """ pass @@ -334,19 +337,19 @@ def get_num_not_departed(self): pass @abstractmethod - def get_fuel_consumption(self, veh_id, error=-1001): - """Return the mpg / s of the specified vehicle. + def get_energy_model(self, veh_id, error=""): + """Return the energy model class object of the specified vehicle. Parameters ---------- veh_id : str or list of str vehicle id, or list of vehicle ids - error : any, optional + error : str value that is returned if the vehicle is not found Returns ------- - float + subclass of BaseEnergyModel """ pass diff --git a/flow/core/kernel/vehicle/traci.py b/flow/core/kernel/vehicle/traci.py index 39bfb35da..948f04cda 100644 --- a/flow/core/kernel/vehicle/traci.py +++ b/flow/core/kernel/vehicle/traci.py @@ -10,6 +10,9 @@ from flow.controllers.car_following_models import SimCarFollowingController from flow.controllers.rlcontroller import RLController from flow.controllers.lane_change_controllers import SimLaneChangeController +from flow.networks import I210SubNetwork +from flow.networks import HighwayNetwork +from flow.controllers.lane_change_controllers import AILaneChangeController from bisect import bisect_left import itertools from copy import deepcopy @@ -59,6 +62,8 @@ def __init__(self, self.num_rl_vehicles = 0 # number of vehicles loaded but not departed vehicles self.num_not_departed = 0 + # number of lane changes in an episode + self.lane_change_count = 0 # contains the parameters associated with each type of vehicle self.type_parameters = {} @@ -71,11 +76,11 @@ def __init__(self, # number of vehicles that entered the network for every time-step self._num_departed = [] - self._departed_ids = 0 + self._departed_ids = [] # number of vehicles to exit the network for every time-step self._num_arrived = [] - self._arrived_ids = 0 + self._arrived_ids = [] self._arrived_rl_ids = [] # whether or not to automatically color vehicles @@ -87,6 +92,15 @@ def __init__(self, # old speeds used to compute accelerations self.previous_speeds = {} + # The time that previous speed is recorded, used to calculate realized_accel + self.previous_time = 0 + + self._is_highway_i210 = \ + isinstance(self.master_kernel.network.network, I210SubNetwork) or \ + isinstance(self.master_kernel.network.network, HighwayNetwork) + + # flag to collect lane leaders/followers/headways/tailways for all + self.collect_info_all = False def initialize(self, vehicles): """Initialize vehicle state information. @@ -135,12 +149,16 @@ def update(self, reset): step """ # copy over the previous speeds - vehicle_obs = {} for veh_id in self.__ids: self.previous_speeds[veh_id] = self.get_speed(veh_id) + # used for lane_count metric + prev_lane = self.get_lane(veh_id) vehicle_obs[veh_id] = \ self.kernel_api.vehicle.getSubscriptionResults(veh_id) + if vehicle_obs[veh_id]: + vehicle_obs[veh_id]['prev_lane'] = prev_lane + sim_obs = self.kernel_api.simulation.getSubscriptionResults() arrived_rl_ids = [] @@ -184,8 +202,8 @@ def update(self, reset): self.prev_last_lc[veh_id] = -float("inf") self._num_departed.clear() self._num_arrived.clear() - self._departed_ids = 0 - self._arrived_ids = 0 + self._departed_ids = [] + self._arrived_ids = [] self._arrived_rl_ids.clear() self.num_not_departed = 0 @@ -220,49 +238,64 @@ def update(self, reset): self.num_not_departed += sim_obs[tc.VAR_LOADED_VEHICLES_NUMBER] - \ sim_obs[tc.VAR_DEPARTED_VEHICLES_NUMBER] - # update the "headway", "leader", and "follower" variables - for veh_id in self.__ids: - try: - _position = vehicle_obs.get(veh_id, {}).get( - tc.VAR_POSITION, -1001) - _angle = vehicle_obs.get(veh_id, {}).get(tc.VAR_ANGLE, -1001) - _time_step = sim_obs[tc.VAR_TIME_STEP] - _time_delta = sim_obs[tc.VAR_DELTA_T] - self.__vehicles[veh_id]["orientation"] = \ - list(_position) + [_angle] - self.__vehicles[veh_id]["timestep"] = _time_step - self.__vehicles[veh_id]["timedelta"] = _time_delta - except TypeError: - print(traceback.format_exc()) - headway = vehicle_obs.get(veh_id, {}).get(tc.VAR_LEADER, None) - # check for a collided vehicle or a vehicle with no leader - if headway is None: - self.__vehicles[veh_id]["leader"] = None - self.__vehicles[veh_id]["follower"] = None - self.__vehicles[veh_id]["headway"] = 1e+3 - self.__vehicles[veh_id]["follower_headway"] = 1e+3 - else: - min_gap = self.minGap[self.get_type(veh_id)] - self.__vehicles[veh_id]["headway"] = headway[1] + min_gap - self.__vehicles[veh_id]["leader"] = headway[0] - if headway[0] in self.__vehicles: - leader = self.__vehicles[headway[0]] - # if veh_id is closer from leader than another follower - # (in case followers are in different converging edges) - if ("follower_headway" not in leader or - headway[1] + min_gap < leader["follower_headway"]): - leader["follower"] = veh_id - leader["follower_headway"] = headway[1] + min_gap + # Update features specific for the pyglet renderer. + pyglet_renderer = self.master_kernel.network.sim_params.render \ + in ['gray', 'dgray', 'rgb', 'drgb'] + if pyglet_renderer: + for veh_id in self.__ids: + try: + _position = vehicle_obs.get(veh_id, {}).get( + tc.VAR_POSITION, -1001) + _angle = vehicle_obs.get(veh_id, {}).get( + tc.VAR_ANGLE, -1001) + _time_step = sim_obs[tc.VAR_TIME_STEP] + _time_delta = sim_obs[tc.VAR_DELTA_T] + self.__vehicles[veh_id]["orientation"] = \ + list(_position) + [_angle] + self.__vehicles[veh_id]["timestep"] = _time_step + self.__vehicles[veh_id]["timedelta"] = _time_delta + except TypeError: + print(traceback.format_exc()) + + # Update the "headway", "leader", and "follower" variables. + if not self._is_highway_i210: + for veh_id in self.__ids: + headway = vehicle_obs.get(veh_id, {}).get(tc.VAR_LEADER, None) + # check for a collided vehicle or a vehicle with no leader + if headway is None: + self.__vehicles[veh_id]["leader"] = None + self.__vehicles[veh_id]["follower"] = None + self.__vehicles[veh_id]["headway"] = 1e+3 + else: + min_gap = self.minGap[self.get_type(veh_id)] + self.__vehicles[veh_id]["headway"] = headway[1] + min_gap + self.__vehicles[veh_id]["leader"] = headway[0] + if headway[0] in self.__vehicles: + leader = self.__vehicles[headway[0]] + # if veh_id is closer from leader than another follower + # (in case followers are in different converging edges) + if ("follower_headway" not in leader or + headway[1] + min_gap < leader["follower_headway"]): + leader["follower"] = veh_id # update the sumo observations variable self.__sumo_obs = vehicle_obs.copy() + # update lane count + for veh_id, vobs in vehicle_obs.items(): + prev_lane = vobs.get('prev_lane', -1001) + if prev_lane != -1001 and self.get_lane(veh_id) != -1001 and prev_lane != self.get_lane(veh_id): + self.lane_change_count += 1 + # update the lane leaders data for each vehicle self._multi_lane_headways() # make sure the rl vehicle list is still sorted self.__rl_ids.sort() + if self._is_highway_i210: + self._update_headways() + def _add_departed(self, veh_id, veh_type): """Add a vehicle that entered the network from an inflow or reset. @@ -287,9 +320,65 @@ def _add_departed(self, veh_id, veh_type): self.num_vehicles += 1 self.__vehicles[veh_id] = dict() + self.set_vehicle_type(veh_id, veh_type) + + # subscribe the new vehicle + self.kernel_api.vehicle.subscribe(veh_id, [ + tc.VAR_LANE_INDEX, tc.VAR_LANEPOSITION, + tc.VAR_ROAD_ID, + tc.VAR_SPEED, + tc.VAR_EDGES, + tc.VAR_POSITION, + tc.VAR_ANGLE, + tc.VAR_SPEED_WITHOUT_TRACI, + tc.VAR_FUELCONSUMPTION, + tc.VAR_DISTANCE + ]) + self.kernel_api.vehicle.subscribeLeader(veh_id, 2000) + + # some constant vehicle parameters to the vehicles class + self.__vehicles[veh_id][ + "length"] = self.kernel_api.vehicle.getLength( + veh_id) + + # set the "last_lc" parameter of the vehicle + self.__vehicles[veh_id]["last_lc"] = -float("inf") + + # get initial state info + self.__sumo_obs[veh_id] = dict() + self.__sumo_obs[veh_id][tc.VAR_ROAD_ID] = \ + self.kernel_api.vehicle.getRoadID(veh_id) + self.__sumo_obs[veh_id][tc.VAR_LANEPOSITION] = \ + self.kernel_api.vehicle.getLanePosition(veh_id) + self.__sumo_obs[veh_id][tc.VAR_LANE_INDEX] = \ + self.kernel_api.vehicle.getLaneIndex(veh_id) + self.__sumo_obs[veh_id][tc.VAR_SPEED] = \ + self.kernel_api.vehicle.getSpeed(veh_id) + self.__sumo_obs[veh_id][tc.VAR_FUELCONSUMPTION] = \ + self.kernel_api.vehicle.getFuelConsumption(veh_id) + + # get the subscription results from the new vehicle + new_obs = self.kernel_api.vehicle.getSubscriptionResults(veh_id) + + return new_obs + + def set_vehicle_type(self, veh_id, veh_type): + """Update/initialize the type of a specific vehicle internally. + + Parameters + ---------- + veh_id : str + the name of the vehicle + veh_type : str + the vehicle type to use + """ # specify the type self.__vehicles[veh_id]["type"] = veh_type + # specify energy model + self.__vehicles[veh_id]["energy_model"] = self.type_parameters[ + veh_type]["energy_model"]() + car_following_params = \ self.type_parameters[veh_type]["car_following_params"] @@ -307,6 +396,11 @@ def _add_departed(self, veh_id, veh_type): self.__vehicles[veh_id]["lane_changer"] = \ lc_controller[0](veh_id=veh_id, **lc_controller[1]) + # if lane changer is AILaneChangeController, set collect info flag True + if lc_controller[0] == AILaneChangeController and \ + not self.collect_info_all: + self.collect_info_all = True + # specify the routing controller class rt_controller = self.type_parameters[veh_type]["routing_controller"] if rt_controller is not None: @@ -317,9 +411,23 @@ def _add_departed(self, veh_id, veh_type): # add the vehicle's id to the list of vehicle ids if accel_controller[0] == RLController: + # Remove from the human-driven lists if it was ever there. Needed + # when updating the vehicle type dynamically. + if veh_id in self.__human_ids: + self.__human_ids.remove(veh_id) + if veh_id in self.__controlled_ids: + self.__controlled_ids.remove(veh_id) + if veh_id in self.__controlled_lc_ids: + self.__controlled_lc_ids.remove(veh_id) + if veh_id not in self.__rl_ids: self.__rl_ids.append(veh_id) else: + # Remove from the RL lists if it was ever there. Needed when + # updating the vehicle type dynamically. + if veh_id in self.__rl_ids: + self.__rl_ids.remove(veh_id) + if veh_id not in self.__human_ids: self.__human_ids.append(veh_id) if accel_controller[0] != SimCarFollowingController: @@ -327,27 +435,6 @@ def _add_departed(self, veh_id, veh_type): if lc_controller[0] != SimLaneChangeController: self.__controlled_lc_ids.append(veh_id) - # subscribe the new vehicle - self.kernel_api.vehicle.subscribe(veh_id, [ - tc.VAR_LANE_INDEX, tc.VAR_LANEPOSITION, - tc.VAR_ROAD_ID, - tc.VAR_SPEED, - tc.VAR_EDGES, - tc.VAR_POSITION, - tc.VAR_ANGLE, - tc.VAR_SPEED_WITHOUT_TRACI, - tc.VAR_FUELCONSUMPTION, - tc.VAR_DISTANCE - ]) - self.kernel_api.vehicle.subscribeLeader(veh_id, 2000) - - # some constant vehicle parameters to the vehicles class - self.__vehicles[veh_id]["length"] = self.kernel_api.vehicle.getLength( - veh_id) - - # set the "last_lc" parameter of the vehicle - self.__vehicles[veh_id]["last_lc"] = -float("inf") - # specify the initial speed self.__vehicles[veh_id]["initial_speed"] = \ self.type_parameters[veh_type]["initial_speed"] @@ -362,38 +449,21 @@ def _add_departed(self, veh_id, veh_type): "lane_change_params"].lane_change_mode self.kernel_api.vehicle.setLaneChangeMode(veh_id, lc_mode) - # get initial state info - self.__sumo_obs[veh_id] = dict() - self.__sumo_obs[veh_id][tc.VAR_ROAD_ID] = \ - self.kernel_api.vehicle.getRoadID(veh_id) - self.__sumo_obs[veh_id][tc.VAR_LANEPOSITION] = \ - self.kernel_api.vehicle.getLanePosition(veh_id) - self.__sumo_obs[veh_id][tc.VAR_LANE_INDEX] = \ - self.kernel_api.vehicle.getLaneIndex(veh_id) - self.__sumo_obs[veh_id][tc.VAR_SPEED] = \ - self.kernel_api.vehicle.getSpeed(veh_id) - self.__sumo_obs[veh_id][tc.VAR_FUELCONSUMPTION] = \ - self.kernel_api.vehicle.getFuelConsumption(veh_id) - # make sure that the order of rl_ids is kept sorted self.__rl_ids.sort() self.num_rl_vehicles = len(self.__rl_ids) - # get the subscription results from the new vehicle - new_obs = self.kernel_api.vehicle.getSubscriptionResults(veh_id) - - return new_obs - def reset(self): """See parent class.""" self.previous_speeds = {} - def remove(self, veh_id): + def remove(self, veh_id, from_sumo=True): """See parent class.""" # remove from sumo - if veh_id in self.kernel_api.vehicle.getIDList(): - self.kernel_api.vehicle.unsubscribe(veh_id) - self.kernel_api.vehicle.remove(veh_id) + if from_sumo: + if veh_id in self.kernel_api.vehicle.getIDList(): + self.kernel_api.vehicle.unsubscribe(veh_id) + self.kernel_api.vehicle.remove(veh_id) if veh_id in self.__ids: self.__ids.remove(veh_id) @@ -540,12 +610,15 @@ def get_num_not_departed(self): """See parent class.""" return self.num_not_departed - def get_fuel_consumption(self, veh_id, error=-1001): - """Return fuel consumption in gallons/s.""" - ml_to_gallons = 0.000264172 + def get_energy_model(self, veh_id, error=""): + """See parent class.""" if isinstance(veh_id, (list, np.ndarray)): - return [self.get_fuel_consumption(vehID, error) for vehID in veh_id] - return self.__sumo_obs.get(veh_id, {}).get(tc.VAR_FUELCONSUMPTION, error) * ml_to_gallons + return [self.get_energy_model(vehID) for vehID in veh_id] + try: + return self.__vehicles.get(veh_id, {'energy_model': error})['energy_model'] + except KeyError: + print("Energy model not specified for vehicle {}".format(veh_id)) + raise def get_previous_speed(self, veh_id, error=-1001): """See parent class.""" @@ -616,6 +689,52 @@ def get_headway(self, veh_id, error=-1001): return [self.get_headway(vehID, error) for vehID in veh_id] return self.__vehicles.get(veh_id, {}).get("headway", error) + def _update_headways(self): + """Update the adjacent vehicle data (for the I210 and highway).""" + # Separate the vehicles by lane. + veh_ids = [[] for _ in range(6)] + for veh_id in self.get_ids(): + veh_ids[self._get_lane(veh_id)].append(veh_id) + + for lane in range(6): + # Sort each vehicle based on the position it is on it's given lane. + sorted_ids = sorted(veh_ids[lane], key=self.get_x_by_id) + + # Update the headways, tailways, leaders, and followers for each + # vehicle. + for i, veh_id in enumerate(sorted_ids): + pos = self.get_x_by_id(veh_id) + + # The front vehicle has no leaders. + if i == len(sorted_ids) - 1: + leader = None + headway = 1000 + else: + leader = sorted_ids[i + 1] + headway = self.get_x_by_id(leader) - pos - 5 + + # The back vehicle have no followers. + if i == 0: + follower = None + else: + follower = sorted_ids[i - 1] + + self.__vehicles[veh_id]["leader"] = leader + self.__vehicles[veh_id]["headway"] = headway + self.__vehicles[veh_id]["follower"] = follower + + def _get_lane(self, veh_id): + """Return a processed lane number.""" + lane = self.get_lane(veh_id) + edge = self.get_edge(veh_id) + return lane + 1 if edge not in [ + "119257908#1-AddedOnRampEdge", + "119257908#1-AddedOffRampEdge", + ":119257908#1-AddedOnRampNode_0", + ":119257908#1-AddedOffRampNode_0", + "119257908#3", + ] else lane + def get_last_lc(self, veh_id, error=-1001): """See parent class.""" if isinstance(veh_id, (list, np.ndarray)): @@ -626,7 +745,7 @@ def get_last_lc(self, veh_id, error=-1001): ' {}.'.format(veh_id, error)) return error else: - return self.__vehicles.get(veh_id, {}).get("headway", error) + return self.__vehicles.get(veh_id, {}).get("last_lc", error) def get_acc_controller(self, veh_id, error=None): """See parent class.""" @@ -751,7 +870,11 @@ def _multi_lane_headways(self): for lane in range(max_lanes): edge_dict[edge][lane].sort(key=lambda x: x[1]) - for veh_id in self.get_rl_ids(): + # get tracked vehicles IDs, for these vehicles info will be collected + tracked_vehs = self.get_ids() if self.collect_info_all \ + else self.get_rl_ids() # FIXME (yf) better collect tracked vehicles + + for veh_id in tracked_vehs: # collect the lane leaders, followers, headways, and tailways for # each vehicle edge = self.get_edge(veh_id) @@ -953,7 +1076,7 @@ def _prev_edge_followers(self, veh_id, edge_dict, lane, num_edges): return tailway, follower - def apply_acceleration(self, veh_ids, acc, smooth=True): + def apply_acceleration(self, veh_ids, acc, smooth_duration=0): """See parent class.""" # to handle the case of a single vehicle if type(veh_ids) == str: @@ -965,8 +1088,8 @@ def apply_acceleration(self, veh_ids, acc, smooth=True): self.__vehicles[vid]["accel"] = acc[i] this_vel = self.get_speed(vid) next_vel = max([this_vel + acc[i] * self.sim_step, 0]) - if smooth: - self.kernel_api.vehicle.slowDown(vid, next_vel, 1e-3) + if smooth_duration: + self.kernel_api.vehicle.slowDown(vid, next_vel, smooth_duration) else: self.kernel_api.vehicle.setSpeed(vid, next_vel) @@ -1023,8 +1146,12 @@ def get_x_by_id(self, veh_id): if self.get_edge(veh_id) == '': # occurs when a vehicle crashes is teleported for some other reason return 0. - return self.master_kernel.network.get_x( - self.get_edge(veh_id), self.get_position(veh_id)) + + if self._is_highway_i210: + return self.get_distance(veh_id) + else: + return self.master_kernel.network.get_x( + self.get_edge(veh_id), self.get_position(veh_id)) def update_vehicle_colors(self): """See parent class. @@ -1137,13 +1264,12 @@ def get_accel(self, veh_id, noise=True, failsafe=True): else: metric_name += '_no_noise' if failsafe: - metric_name += '_with_falsafe' + metric_name += '_with_failsafe' else: metric_name += '_no_failsafe' - if metric_name not in self.__vehicles[veh_id]: - self.__vehicles[veh_id][metric_name] = None - return self.__vehicles[veh_id][metric_name] + return self.__vehicles[veh_id].get(metric_name, None) \ + or self.get_realized_accel(veh_id) def update_accel(self, veh_id, accel, noise=True, failsafe=True): """See parent class.""" @@ -1153,7 +1279,7 @@ def update_accel(self, veh_id, accel, noise=True, failsafe=True): else: metric_name += '_no_noise' if failsafe: - metric_name += '_with_falsafe' + metric_name += '_with_failsafe' else: metric_name += '_no_failsafe' diff --git a/flow/core/params.py b/flow/core/params.py index 79ad8d689..a79bb751e 100755 --- a/flow/core/params.py +++ b/flow/core/params.py @@ -7,6 +7,10 @@ from flow.controllers.car_following_models import SimCarFollowingController from flow.controllers.rlcontroller import RLController from flow.controllers.lane_change_controllers import SimLaneChangeController +from flow.energy_models.toyota_energy import PriusEnergy +from flow.energy_models.toyota_energy import TacomaEnergy +from flow.energy_models.power_demand import PDMCombustionEngine +from flow.energy_models.power_demand import PDMElectric SPEED_MODES = { @@ -39,6 +43,9 @@ "only_right_drive_safe": 576 } +ENERGY_MODELS = set([PriusEnergy, TacomaEnergy, PDMCombustionEngine, PDMElectric]) +DEFAULT_ENERGY_MODEL = PDMCombustionEngine + # Traffic light defaults PROGRAM_ID = 1 MAX_GAP = 3.0 @@ -262,6 +269,7 @@ def add(self, num_vehicles=0, car_following_params=None, lane_change_params=None, + energy_model=DEFAULT_ENERGY_MODEL, color=None): """Add a sequence of vehicles to the list of vehicles in the network. @@ -298,6 +306,12 @@ def add(self, # FIXME: depends on simulator lane_change_params = SumoLaneChangeParams() + if energy_model not in ENERGY_MODELS: + print('{} for vehicle {} is not a valid energy model. Defaulting to {}\n'.format(energy_model, + veh_id, + DEFAULT_ENERGY_MODEL)) + energy_model = DEFAULT_ENERGY_MODEL + type_params = {} type_params.update(car_following_params.controller_params) type_params.update(lane_change_params.controller_params) @@ -311,7 +325,8 @@ def add(self, "routing_controller": routing_controller, "initial_speed": initial_speed, "car_following_params": car_following_params, - "lane_change_params": lane_change_params} + "lane_change_params": lane_change_params, + "energy_model": energy_model} if color: type_params['color'] = color @@ -334,7 +349,9 @@ def add(self, "car_following_params": car_following_params, "lane_change_params": - lane_change_params + lane_change_params, + "energy_model": + energy_model }) # This is used to return the actual headways from the vehicles class. @@ -411,6 +428,15 @@ class SimParams(object): specifies rendering resolution (pixel / meter) force_color_update : bool, optional whether or not to automatically color vehicles according to their types + save_state_time : float, optional + the time to save the simulation state. States can then be reloaded + later. If set to None, no initial state is saved. + save_state_file : str, optional + the path to save the initial state to. Only used is save_state_time is + not None. + load_state : str, optional + the path to the initial state that the network should be initialized + with """ def __init__(self, @@ -422,7 +448,10 @@ def __init__(self, sight_radius=25, show_radius=False, pxpm=2, - force_color_update=False): + force_color_update=False, + save_state_time=None, + save_state_file="initial_state.xml", + load_state=None): """Instantiate SimParams.""" self.sim_step = sim_step self.render = render @@ -433,6 +462,9 @@ def __init__(self, self.pxpm = pxpm self.show_radius = show_radius self.force_color_update = force_color_update + self.save_state_time = save_state_time + self.save_state_file = save_state_file + self.load_state = load_state class AimsunParams(SimParams): @@ -588,6 +620,17 @@ class SumoParams(SimParams): current time step use_ballistic: bool, optional If true, use a ballistic integration step instead of an euler step + disable_collisions: bool, optional + If true, disables explicit collision checking and teleporting in SUMO + save_state_time : float, optional + the time to save the simulation state. States can then be reloaded + later. If set to None, no initial state is saved. + save_state_file : str, optional + the path to save the initial state to. Only used is save_state_time is + not None. + load_state : str, optional + the path to the initial state that the network should be initialized + with """ def __init__(self, @@ -609,11 +652,16 @@ def __init__(self, teleport_time=-1, num_clients=1, color_by_speed=False, - use_ballistic=False): + use_ballistic=False, + disable_collisions=False, + save_state_time=None, + save_state_file="initial_state.xml", + load_state=None): """Instantiate SumoParams.""" super(SumoParams, self).__init__( sim_step, render, restart_instance, emission_path, save_render, - sight_radius, show_radius, pxpm, force_color_update) + sight_radius, show_radius, pxpm, force_color_update, + save_state_time, save_state_file, load_state) self.port = port self.lateral_resolution = lateral_resolution self.no_step_log = no_step_log @@ -624,6 +672,7 @@ def __init__(self, self.num_clients = num_clients self.color_by_speed = color_by_speed self.use_ballistic = use_ballistic + self.disable_collisions = disable_collisions class EnvParams: @@ -657,6 +706,9 @@ class EnvParams: specifies whether to clip actions from the policy by their range when they are inputted to the reward function. Note that the actions are still clipped before they are provided to `apply_rl_actions`. + done_at_exit : bool, optional + If true, done is returned as True when the vehicle exits. This is only + applied to multi-agent environments. """ def __init__(self, @@ -665,7 +717,8 @@ def __init__(self, warmup_steps=0, sims_per_step=1, evaluate=False, - clip_actions=True): + clip_actions=True, + done_at_exit=True): """Instantiate EnvParams.""" self.additional_params = \ additional_params if additional_params is not None else {} @@ -674,6 +727,7 @@ def __init__(self, self.sims_per_step = sims_per_step self.evaluate = evaluate self.clip_actions = clip_actions + self.done_at_exit = done_at_exit def get_additional_param(self, key): """Return a variable from additional_params.""" diff --git a/flow/core/rewards.py b/flow/core/rewards.py index 3cca916f5..20ed1c6a7 100755 --- a/flow/core/rewards.py +++ b/flow/core/rewards.py @@ -307,58 +307,26 @@ def punish_rl_lane_changes(env, penalty=1): def energy_consumption(env, gain=.001): - """Calculate power consumption of a vehicle. + """Calculate power consumption for all vehicle. Assumes vehicle is an average sized vehicle. The power calculated here is the lower bound of the actual power consumed by a vehicle. - """ - power = 0 - - M = 1200 # mass of average sized vehicle (kg) - g = 9.81 # gravitational acceleration (m/s^2) - Cr = 0.005 # rolling resistance coefficient - Ca = 0.3 # aerodynamic drag coefficient - rho = 1.225 # air density (kg/m^3) - A = 2.6 # vehicle cross sectional area (m^2) - for veh_id in env.k.vehicle.get_ids(): - speed = env.k.vehicle.get_speed(veh_id) - prev_speed = env.k.vehicle.get_previous_speed(veh_id) - - accel = abs(speed - prev_speed) / env.sim_step - power += M * speed * accel + M * g * Cr * speed + 0.5 * rho * A * Ca * speed ** 3 - - return -gain * power - - -def veh_energy_consumption(env, veh_id, gain=.001): - """Calculate power consumption of a vehicle. - - Assumes vehicle is an average sized vehicle. - The power calculated here is the lower bound of the actual power consumed - by a vehicle. + Parameters + ---------- + env : flow.envs.Env + the environment variable, which contains information on the current + state of the system. + gain : float + scaling factor for the reward """ - power = 0 - - M = 1200 # mass of average sized vehicle (kg) - g = 9.81 # gravitational acceleration (m/s^2) - Cr = 0.005 # rolling resistance coefficient - Ca = 0.3 # aerodynamic drag coefficient - rho = 1.225 # air density (kg/m^3) - A = 2.6 # vehicle cross sectional area (m^2) - speed = env.k.vehicle.get_speed(veh_id) - prev_speed = env.k.vehicle.get_previous_speed(veh_id) - - accel = abs(speed - prev_speed) / env.sim_step - - power += M * speed * accel + M * g * Cr * speed + 0.5 * rho * A * Ca * speed ** 3 - - return -gain * power + veh_ids = env.k.vehicle.get_ids() + return veh_energy_consumption(env, veh_ids, gain) -def miles_per_megajoule(env, veh_ids=None, gain=.001): - """Calculate miles per mega-joule of either a particular vehicle or the total average of all the vehicles. +def veh_energy_consumption(env, veh_ids=None, gain=.001): + """Calculate power consumption of a vehicle. Assumes vehicle is an average sized vehicle. The power calculated here is the lower bound of the actual power consumed @@ -369,70 +337,64 @@ def miles_per_megajoule(env, veh_ids=None, gain=.001): env : flow.envs.Env the environment variable, which contains information on the current state of the system. - veh_ids : [list] - list of veh_ids to compute the reward over + veh_ids : [list] or str + list of veh_ids or single veh_id to compute the reward over gain : float scaling factor for the reward """ - mpj = 0 - counter = 0 if veh_ids is None: veh_ids = env.k.vehicle.get_ids() elif not isinstance(veh_ids, list): veh_ids = [veh_ids] - for veh_id in veh_ids: - speed = env.k.vehicle.get_speed(veh_id) - # convert to be positive since the function called is a penalty - power = -veh_energy_consumption(env, veh_id, gain=1.0) - if power > 0 and speed >= 0.0: - counter += 1 - # meters / joule is (v * \delta t) / (power * \delta t) - mpj += speed / power - if counter > 0: - mpj /= counter - # convert from meters per joule to miles per joule - mpj /= 1609.0 - # convert from miles per joule to miles per megajoule - mpj *= 10**6 + power = 0 + for veh_id in veh_ids: + if veh_id not in env.k.vehicle.previous_speeds: + continue + energy_model = env.k.vehicle.get_energy_model(veh_id) + if energy_model != "": + speed = env.k.vehicle.get_speed(veh_id) + accel = env.k.vehicle.get_accel(veh_id, noise=False, failsafe=True) + grade = env.k.vehicle.get_road_grade(veh_id) + power += energy_model.get_instantaneous_power(accel, speed, grade) - return mpj * gain + return -gain * power -def miles_per_gallon(env, veh_ids=None, gain=.001): - """Calculate mpg of either a particular vehicle or the total average of all the vehicles. - - Assumes vehicle is an average sized vehicle. - The power calculated here is the lower bound of the actual power consumed - by a vehicle. +def instantaneous_mpg(env, veh_ids=None, gain=.001): + """Calculate the instantaneous mpg for every simulation step specific to the vehicle type. Parameters ---------- env : flow.envs.Env the environment variable, which contains information on the current state of the system. - veh_ids : [list] - list of veh_ids to compute the reward over + veh_ids : [list] or str + list of veh_ids or single veh_id to compute the reward over gain : float scaling factor for the reward """ - mpg = 0 - counter = 0 if veh_ids is None: veh_ids = env.k.vehicle.get_ids() elif not isinstance(veh_ids, list): veh_ids = [veh_ids] + + cumulative_gallons = 0 + cumulative_distance = 0 for veh_id in veh_ids: - speed = env.k.vehicle.get_speed(veh_id) - gallons_per_s = env.k.vehicle.get_fuel_consumption(veh_id) - if gallons_per_s > 0 and speed >= 0.0: - counter += 1 - # meters / gallon is (v * \delta t) / (gallons_per_s * \delta t) - mpg += speed / gallons_per_s - if counter > 0: - mpg /= counter - - # convert from meters per gallon to miles per gallon - mpg /= 1609.0 + energy_model = env.k.vehicle.get_energy_model(veh_id) + if energy_model != "": + speed = env.k.vehicle.get_speed(veh_id) + accel = env.k.vehicle.get_accel(veh_id, noise=False, failsafe=True) + grade = env.k.vehicle.get_road_grade(veh_id) + gallons_per_hr = energy_model.get_instantaneous_fuel_consumption(accel, speed, grade) + if speed >= 0.0: + cumulative_gallons += gallons_per_hr + cumulative_distance += speed + + cumulative_gallons /= 3600.0 + cumulative_distance /= 1609.34 + # miles / gallon is (distance_dot * \delta t) / (gallons_dot * \delta t) + mpg = cumulative_distance / (cumulative_gallons + 1e-6) return mpg * gain diff --git a/flow/core/util.py b/flow/core/util.py index 1821a76a5..c0c31f811 100755 --- a/flow/core/util.py +++ b/flow/core/util.py @@ -4,7 +4,6 @@ import errno import os from lxml import etree -from xml.etree import ElementTree def makexml(name, nsl): @@ -47,42 +46,39 @@ def emission_to_csv(emission_path, output_path=None): path to the csv file that will be generated, default is the same directory as the emission file, with the same name """ - parser = etree.XMLParser(recover=True) - tree = ElementTree.parse(emission_path, parser=parser) - root = tree.getroot() - - # parse the xml data into a dict + context = etree.iterparse(emission_path, recover=True) out_data = [] - for time in root.findall('timestep'): - t = float(time.attrib['time']) - - for car in time: - out_data.append(dict()) - try: - out_data[-1]['time'] = t - out_data[-1]['CO'] = float(car.attrib['CO']) - out_data[-1]['y'] = float(car.attrib['y']) - out_data[-1]['CO2'] = float(car.attrib['CO2']) - out_data[-1]['electricity'] = float(car.attrib['electricity']) - out_data[-1]['type'] = car.attrib['type'] - out_data[-1]['id'] = car.attrib['id'] - out_data[-1]['eclass'] = car.attrib['eclass'] - out_data[-1]['waiting'] = float(car.attrib['waiting']) - out_data[-1]['NOx'] = float(car.attrib['NOx']) - out_data[-1]['fuel'] = float(car.attrib['fuel']) - out_data[-1]['HC'] = float(car.attrib['HC']) - out_data[-1]['x'] = float(car.attrib['x']) - out_data[-1]['route'] = car.attrib['route'] - out_data[-1]['relative_position'] = float(car.attrib['pos']) - out_data[-1]['noise'] = float(car.attrib['noise']) - out_data[-1]['angle'] = float(car.attrib['angle']) - out_data[-1]['PMx'] = float(car.attrib['PMx']) - out_data[-1]['speed'] = float(car.attrib['speed']) - out_data[-1]['edge_id'] = car.attrib['lane'].rpartition('_')[0] - out_data[-1]['lane_number'] = car.attrib['lane'].\ - rpartition('_')[-1] - except KeyError: - del out_data[-1] + for event, elem in context: + if elem.tag == "timestep": + t = float(elem.attrib['time']) + for car in elem: + out_data.append(dict()) + try: + out_data[-1]['time'] = t + out_data[-1]['CO'] = float(car.attrib['CO']) + out_data[-1]['y'] = float(car.attrib['y']) + out_data[-1]['CO2'] = float(car.attrib['CO2']) + out_data[-1]['electricity'] = float(car.attrib['electricity']) + out_data[-1]['type'] = car.attrib['type'] + out_data[-1]['id'] = car.attrib['id'] + out_data[-1]['eclass'] = car.attrib['eclass'] + out_data[-1]['waiting'] = float(car.attrib['waiting']) + out_data[-1]['NOx'] = float(car.attrib['NOx']) + out_data[-1]['fuel'] = float(car.attrib['fuel']) + out_data[-1]['HC'] = float(car.attrib['HC']) + out_data[-1]['x'] = float(car.attrib['x']) + out_data[-1]['route'] = car.attrib['route'] + out_data[-1]['relative_position'] = float(car.attrib['pos']) + out_data[-1]['noise'] = float(car.attrib['noise']) + out_data[-1]['angle'] = float(car.attrib['angle']) + out_data[-1]['PMx'] = float(car.attrib['PMx']) + out_data[-1]['speed'] = float(car.attrib['speed']) + out_data[-1]['edge_id'] = car.attrib['lane'].rpartition('_')[0] + out_data[-1]['lane_number'] = car.attrib['lane']. \ + rpartition('_')[-1] + except KeyError: + del out_data[-1] + elem.clear() # sort the elements of the dictionary by the vehicle id out_data = sorted(out_data, key=lambda k: k['id']) diff --git a/flow/data_pipeline/README.md b/flow/data_pipeline/README.md new file mode 100644 index 000000000..65aeb8d49 --- /dev/null +++ b/flow/data_pipeline/README.md @@ -0,0 +1,12 @@ +To run a simulation with output stored locally only: + + `python simulate.py EXP_CONFIG --gen_emission` + +To run a simulation and upload output to pipeline: + + `python simulate.py EXP_CONFIG --to_aws` + +To run a simulation, upload output to pipeline, and mark it as baseline: + + `python simulate.py EXP_CONFIG --to_aws --is_baseline` + diff --git a/flow/data_pipeline/__init__.py b/flow/data_pipeline/__init__.py new file mode 100644 index 000000000..d9d6a6573 --- /dev/null +++ b/flow/data_pipeline/__init__.py @@ -0,0 +1 @@ +"""Empty init file to ensure that data_pipeline is recognized as a package.""" diff --git a/flow/data_pipeline/data_pipeline.py b/flow/data_pipeline/data_pipeline.py new file mode 100644 index 000000000..587ce6e70 --- /dev/null +++ b/flow/data_pipeline/data_pipeline.py @@ -0,0 +1,469 @@ +"""contains class and helper functions for the data pipeline.""" +import pandas as pd +import boto3 +from botocore.exceptions import ClientError +from flow.data_pipeline.query import QueryStrings, prerequisites, tables +from time import time +from datetime import date +import csv +from io import StringIO +import json +import collections + + +def generate_trajectory_table(emission_files, trajectory_table_path, source_id): + """Generate desired output for the trajectory_table based on SUMO emissions. + + Parameters + ---------- + emission_files : list + paths to the SUMO emission + trajectory_table_path : str + path to the file for S3 upload only + source_id : str + a unique id for the simulation that generate these emissions + """ + for i in range(len(emission_files)): + # 1000000 rows are approximately 260 MB, which is an appropriate size to load into memory at once + emission_output = pd.read_csv(emission_files[i], iterator=True, chunksize=1000000) + chunk_count = 0 + for chunk in emission_output: + chunk['source_id'] = source_id + chunk['run_id'] = "run_{}".format(i) + # add header row to the file only at the first run (when i==0) and the first chunk (chunk_count==0) + chunk.to_csv(trajectory_table_path, mode='a+', index=False, header=(chunk_count == 0) and (i == 0)) + chunk_count += 1 + + +def write_dict_to_csv(data_path, extra_info, include_header=False): + """Write extra to the CSV file at data_path, create one if not exist. + + Parameters + ---------- + data_path : str + output file path + extra_info: dict + extra information needed in the trajectory table, collected from flow + include_header: bool + whether or not to include the header in the output, this should be set to + True for the first write to the a empty or newly created CSV, and set to + False for subsequent appends. + """ + extra_info = pd.DataFrame.from_dict(extra_info) + extra_info.to_csv(data_path, mode='a+', index=False, header=include_header) + + +def upload_to_s3(bucket_name, bucket_key, file_path, metadata={}): + """Upload a file to S3 bucket. + + Parameters + ---------- + bucket_name : str + the bucket to upload to + bucket_key: str + the key within the bucket for the file + file_path: str + the path of the file to be uploaded + metadata: dict + all the metadata that should be attached to this simulation + """ + s3 = boto3.resource("s3") + s3.Bucket(bucket_name).upload_file(file_path, bucket_key, + ExtraArgs={"Metadata": metadata}) + return + + +def get_extra_info(veh_kernel, extra_info, veh_ids, source_id, run_id): + """Get all the necessary information for the trajectory output from flow.""" + for vid in veh_ids: + extra_info["time_step"].append(veh_kernel.get_timestep(vid) / 1000) + extra_info["id"].append(vid) + position = veh_kernel.get_2d_position(vid) + extra_info["x"].append(position[0]) + extra_info["y"].append(position[1]) + extra_info["speed"].append(veh_kernel.get_speed(vid)) + extra_info["headway"].append(veh_kernel.get_headway(vid)) + extra_info["leader_id"].append(veh_kernel.get_leader(vid)) + extra_info["follower_id"].append(veh_kernel.get_follower(vid)) + extra_info["leader_rel_speed"].append(veh_kernel.get_speed( + veh_kernel.get_leader(vid)) - veh_kernel.get_speed(vid)) + extra_info["target_accel_with_noise_with_failsafe"].append(veh_kernel.get_accel(vid)) + extra_info["target_accel_no_noise_no_failsafe"].append( + veh_kernel.get_accel(vid, noise=False, failsafe=False)) + extra_info["target_accel_with_noise_no_failsafe"].append( + veh_kernel.get_accel(vid, noise=True, failsafe=False)) + extra_info["target_accel_no_noise_with_failsafe"].append( + veh_kernel.get_accel(vid, noise=False, failsafe=True)) + extra_info["realized_accel"].append(veh_kernel.get_realized_accel(vid)) + extra_info["road_grade"].append(veh_kernel.get_road_grade(vid)) + extra_info["edge_id"].append(veh_kernel.get_edge(vid)) + extra_info["lane_id"].append(veh_kernel.get_lane(vid)) + extra_info["distance"].append(veh_kernel.get_distance(vid)) + extra_info["relative_position"].append(veh_kernel.get_position(vid)) + extra_info["source_id"].append(source_id) + extra_info["run_id"].append(run_id) + + +def get_configuration(submitter_name=None, strategy_name=None): + """Get configuration for the metadata table.""" + try: + config_df = pd.read_csv('./data_pipeline_config') + except FileNotFoundError: + config_df = pd.DataFrame(data={"submitter_name": [""], "strategy": [""]}) + + if not config_df['submitter_name'][0]: + if submitter_name: + name = submitter_name + else: + name = input("Please enter your name:").strip() + while not name: + name = input("Please enter a non-empty name:").strip() + config_df['submitter_name'] = [name] + + if strategy_name: + strategy = strategy_name + else: + strategy = input( + "Please enter strategy name (current: \"{}\"):".format(config_df["strategy"][0])).strip() + if strategy: + config_df['strategy'] = [strategy] + + config_df.to_csv('./data_pipeline_config', index=False) + + return config_df['submitter_name'][0], config_df['strategy'][0] + + +def delete_obsolete_data(s3, latest_key, table, bucket="circles.data.pipeline"): + """Delete the obsolete data on S3.""" + keys = list_object_keys(s3, bucket=bucket, prefixes=table, suffix='.csv') + keys.remove(latest_key) + for key in keys: + s3.delete_object(Bucket=bucket, Key=key) + + +def update_baseline(s3, baseline_network, baseline_source_id): + """Update the baseline table on S3 if new baseline run is added.""" + obj = s3.get_object(Bucket='circles.data.pipeline', Key='baseline_table/baselines.csv')['Body'] + original_str = obj.read().decode() + reader = csv.DictReader(StringIO(original_str)) + new_str = StringIO() + writer = csv.DictWriter(new_str, fieldnames=['network', 'source_id']) + writer.writeheader() + writer.writerow({'network': baseline_network, 'source_id': baseline_source_id}) + for row in reader: + if row['network'] != baseline_network: + writer.writerow(row) + s3.put_object(Bucket='circles.data.pipeline', Key='baseline_table/baselines.csv', + Body=new_str.getvalue().replace('\r', '').encode()) + + +def get_completed_queries(s3, source_id): + """Return the deserialized set of completed queries from S3.""" + try: + completed_queries_obj = \ + s3.get_object(Bucket='circles.data.pipeline', Key='lambda_temp/{}'.format(source_id))['Body'] + completed_queries = json.loads(completed_queries_obj.read().decode('utf-8')) + except ClientError as e: + if e.response['Error']['Code'] == 'NoSuchKey': + completed_queries = set() + else: + raise + return set(completed_queries) + + +def put_completed_queries(s3, source_id, completed_queries_set): + """Put the completed queries list into S3 as in a serialized json format.""" + completed_queries_list = list(completed_queries_set) + completed_queries_json = json.dumps(completed_queries_list) + s3.put_object(Bucket='circles.data.pipeline', Key='lambda_temp/{}'.format(source_id), + Body=completed_queries_json.encode('utf-8')) + + +def get_ready_queries(completed_queries, new_query): + """Return queries whose prerequisite queries are completed.""" + readied_queries = [] + unfinished_queries = prerequisites.keys() - completed_queries + upadted_completed_queries = completed_queries.copy() + upadted_completed_queries.add(new_query) + for query_name in unfinished_queries: + if not prerequisites[query_name][1].issubset(completed_queries): + if prerequisites[query_name][1].issubset(upadted_completed_queries): + readied_queries.append((query_name, prerequisites[query_name][0])) + return readied_queries + + +def list_object_keys(s3, bucket='circles.data.pipeline', prefixes='', suffix=''): + """Return all keys in the given bucket that start with prefix and end with suffix. Not limited by 1000.""" + contents = [] + if not isinstance(prefixes, collections.Iterable) or type(prefixes) is str: + prefixes = [prefixes] + for prefix in prefixes: + response = s3.list_objects_v2(Bucket=bucket, Prefix=prefix) + if 'Contents' in response: + contents.extend(response['Contents']) + while response['IsTruncated']: + response = s3.list_objects_v2(Bucket=bucket, Prefix=prefix, + ContinuationToken=response['NextContinuationToken']) + contents.extend(response['Contents']) + keys = [content['Key'] for content in contents if content['Key'].endswith(suffix)] + return keys + + +def delete_table(s3, bucket='circles.data.pipeline', only_query_result=True, table='', source_id=''): + """Delete the specified data files in S3.""" + queries = ["lambda_temp"] + if table: + queries.append(table) + else: + queries = tables + if only_query_result: + queries.remove('fact_vehicle_trace') + queries.remove('metadata_table') + else: + confirmation = input("Are you sure you want to delete the simulation emission file? This process is" + "irreversible. (Y/N)").strip() + if not (confirmation in ['y', 'Y', 'yes', 'Yes']): + return + if source_id: + queries.remove('leaderboard_chart_agg') + queries.remove('fact_top_scores') + keys = list_object_keys(s3, bucket=bucket, prefixes=queries) + if source_id: + keys = [key for key in keys if source_id in key] + for key in keys: + s3.delete_object(Bucket=bucket, Key=key) + + +def rerun_query(s3, queue_url, bucket='circles.data.pipeline', source_id=''): + """Re-run queries for simulation datas that has been uploaded to s3, will delete old data before re-run.""" + vehicle_trace_keys = list_object_keys(s3, bucket=bucket, prefixes="fact_vehicle_trace", suffix='.csv') + delete_table(s3, bucket=bucket, source_id=source_id) + if source_id: + vehicle_trace_keys = [key for key in vehicle_trace_keys if source_id in key] + sqs_client = boto3.client('sqs') + # A s3 put event message template, used to trigger lambda for an existing submission without uploading it again + event_template = """ + {{ + "Records": [ + {{ + "eventVersion": "2.0", + "eventSource": "aws:s3", + "awsRegion": "us-west-2", + "eventTime": "1970-01-01T00:00:00.000Z", + "eventName": "ObjectCreated:Put", + "userIdentity": {{ + "principalId": "EXAMPLE" + }}, + "requestParameters": {{ + "sourceIPAddress": "127.0.0.1" + }}, + "responseElements": {{ + "x-amz-request-id": "EXAMPLE123456789", + "x-amz-id-2": "EXAMPLE123/5678abcdefghijklambdaisawesome/mnopqrstuvwxyzABCDEFGH" + }}, + "s3": {{ + "s3SchemaVersion": "1.0", + "configurationId": "testConfigRule", + "bucket": {{ + "name": "{bucket}", + "ownerIdentity": {{ + "principalId": "EXAMPLE" + }}, + "arn": "arn:aws:s3:::{bucket}" + }}, + "object": {{ + "key": "{key}", + "size": 1024, + "eTag": "0123456789abcdef0123456789abcdef", + "sequencer": "0A1B2C3D4E5F678901" + }} + }} + }} + ] + }}""" + for key in vehicle_trace_keys: + sqs_client.send_message(QueueUrl=queue_url, + MessageBody=event_template.format(bucket=bucket, key=key)) + + +def list_source_ids(s3, bucket='circles.data.pipeline'): + """Return a list of the source_id of all simulations which has been uploaded to s3.""" + vehicle_trace_keys = list_object_keys(s3, bucket=bucket, prefixes="fact_vehicle_trace", suffix='csv') + source_ids = ['flow_{}'.format(key.split('/')[2].split('=')[1].split('_')[1]) for key in vehicle_trace_keys] + return source_ids + + +def collect_metadata_from_config(config_obj): + """Collect the metadata from the exp_config files.""" + supplied_metadata = dict() + supplied_metadata['version'] = [getattr(config_obj, 'VERSION', '2.0')] + supplied_metadata['on_ramp'] = [str(getattr(config_obj, 'ON_RAMP', False))] + supplied_metadata['penetration_rate'] = [str(100.0 * getattr(config_obj, 'PENETRATION_RATE', 0.0))] + supplied_metadata['road_grade'] = [str(getattr(config_obj, 'ROAD_GRADE', False))] + return supplied_metadata + + +class AthenaQuery: + """Class used to run queries. + + Act as a query engine, maintains an open session with AWS Athena. + + Attributes + ---------- + MAX_WAIT : int + maximum number of seconds to wait before declares time-out + client : boto3.client + the athena client that is used to run the query + existing_partitions : list + a list of partitions that is already recorded in Athena's datalog, + this is obtained through query at the initialization of this class + instance. + """ + + def __init__(self): + """Initialize AthenaQuery instance. + + initialize a client session with AWS Athena, + query Athena to obtain extisting_partition. + """ + self.MAX_WAIT = 60 + self.client = boto3.client("athena") + self.existing_partitions = {} + + def get_existing_partitions(self, table): + """Return the existing partitions in the S3 bucket. + + Returns + ------- + partitions: a list of existing partitions on S3 bucket + """ + response = self.client.start_query_execution( + QueryString='SHOW PARTITIONS {}'.format(table), + QueryExecutionContext={ + 'Database': 'circles' + }, + WorkGroup='primary' + ) + if self.wait_for_execution(response['QueryExecutionId']): + raise RuntimeError("get current partitions timed out") + response = self.client.get_query_results( + QueryExecutionId=response['QueryExecutionId'], + MaxResults=1000 + ) + return [data['Data'][0]['VarCharValue'] for data in response['ResultSet']['Rows']] + + def check_status(self, execution_id): + """Return the status of the execution with given id. + + Parameters + ---------- + execution_id : string + id of the execution that is checked for + Returns + ------- + status: str + QUEUED|RUNNING|SUCCEEDED|FAILED|CANCELLED + """ + response = self.client.get_query_execution( + QueryExecutionId=execution_id + ) + return response['QueryExecution']['Status']['State'] + + def wait_for_execution(self, execution_id): + """Wait for the execution to finish or time-out. + + Parameters + ---------- + execution_id : str + id of the execution this is watiing for + Returns + ------- + time_out: bool + True if time-out, False if success + Raises + ------ + RuntimeError: if execution failed or get canceled + """ + start = time() + while time() - start < self.MAX_WAIT: + state = self.check_status(execution_id) + if state == 'FAILED' or state == 'CANCELLED': + raise RuntimeError("update partition failed") + elif state == 'SUCCEEDED': + return False + return True + + def update_partition(self, table, submission_date, partition): + """Load the given partition to the trajectory_table on Athena. + + Parameters + ---------- + table : str + the name of the table to update + submission_date : str + the new partition date that needs to be loaded + partition : str + the new partition that needs to be loaded + """ + response = self.client.start_query_execution( + QueryString=QueryStrings['UPDATE_PARTITION'].value.format(table=table, date=submission_date, + partition=partition), + QueryExecutionContext={ + 'Database': 'circles' + }, + WorkGroup='primary' + ) + if self.wait_for_execution(response['QueryExecutionId']): + raise RuntimeError("update partition timed out") + self.existing_partitions[table].append("date={}/partition_name={}".format(submission_date, partition)) + return + + def repair_partition(self, table, submission_date, partition): + """Load the missing partitions.""" + if table not in self.existing_partitions.keys(): + self.existing_partitions[table] = self.get_existing_partitions(table) + if "date={}/partition_name={}".format(submission_date, partition) not in \ + self.existing_partitions[table]: + self.update_partition(table, submission_date, partition) + + def run_query(self, query_name, result_location="s3://circles.data.pipeline/result/", + submission_date="today", partition="default", **kwargs): + """Start the execution of a query, does not wait for it to finish. + + Parameters + ---------- + query_name : str + name of the query in QueryStrings enum that will be run + result_location: str, optional + location on the S3 bucket where the result will be stored + submission_date : str + name of the partition date to run this query on + partition: str, optional + name of the partition to run this query on + Returns + ------- + execution_id: str + the execution id of the execution started by this method + Raises + ------ + ValueError: if tries to run a query not existed in QueryStrings enum + """ + if query_name not in QueryStrings.__members__: + raise ValueError("query not existed: please add it to query.py") + + if submission_date == "today": + submission_date = date.today().isoformat() + + source_id = "flow_{}".format(partition.split('_')[1]) + + response = self.client.start_query_execution( + QueryString=QueryStrings[query_name].value.format(date=submission_date, partition=source_id, **kwargs), + QueryExecutionContext={ + 'Database': 'circles' + }, + ResultConfiguration={ + 'OutputLocation': result_location, + }, + WorkGroup='primary' + ) + return response['QueryExecutionId'] diff --git a/flow/data_pipeline/lambda_function.py b/flow/data_pipeline/lambda_function.py new file mode 100644 index 000000000..d82e6e341 --- /dev/null +++ b/flow/data_pipeline/lambda_function.py @@ -0,0 +1,89 @@ +"""lambda function on AWS Lambda.""" +import boto3 +import json +from urllib.parse import unquote_plus +from flow.data_pipeline.data_pipeline import AthenaQuery, delete_obsolete_data, update_baseline, \ + get_ready_queries, get_completed_queries, put_completed_queries +from flow.data_pipeline.query import tables, network_filters, summary_tables, triggers, max_decel, leader_max_decel + +s3 = boto3.client('s3') +queryEngine = AthenaQuery() +sqs = boto3.client('sqs') + + +def lambda_handler(event, context): + """Handle S3 put event on AWS Lambda.""" + # stores all lists of completed query for each source_id + completed = {} + records = [] + event_records = [] + # do a pre-sweep to put all s3 records in one list + for event_record in event['Records']: + if event_record["eventSource"] == "aws:s3": + event_records.append(event_record) + elif event_record['eventSource'] == "aws:sqs": + s3_event = json.loads(event_record['body']) + event_records.extend(s3_event['Records']) + # do a pre-sweep to handle tasks other than initalizing a query + for record in event_records: + bucket = record['s3']['bucket']['name'] + key = unquote_plus(record['s3']['object']['key']) + table = key.split('/')[0] + if table not in tables: + continue + # delete unwanted metadata files + s3.delete_object(Bucket=bucket, Key=(key + '.metadata')) + # load the partition for newly added table + query_date = key.split('/')[-3].split('=')[-1] + partition = key.split('/')[-2].split('=')[-1] + source_id = "flow_{}".format(partition.split('_')[1]) + if table == "fact_vehicle_trace": + query_name = "FACT_VEHICLE_TRACE" + else: + query_name = partition.replace(source_id, "")[1:] + queryEngine.repair_partition(table, query_date, partition) + # delete obsolete data + if table in summary_tables: + delete_obsolete_data(s3, key, table) + # add table that need to start a query to list + if query_name in triggers: + records.append((bucket, key, table, query_name, query_date, partition, source_id)) + + # initialize the queries + for bucket, key, table, query_name, query_date, partition, source_id in records: + # retrieve the set of completed query for this source_id if not already available + if source_id not in completed.keys(): + completed[source_id] = get_completed_queries(s3, source_id) + # if query already recorded before, skip it. This is to tolerate repetitive execution by Lambda + if query_name in completed[source_id]: + continue + # retrieve metadata and use it to determine the right location filters + metadata_key = "fact_vehicle_trace/date={0}/partition_name={1}/{1}.csv".format(query_date, source_id) + response = s3.head_object(Bucket=bucket, Key=metadata_key) + if 'network' in response["Metadata"]: + network = response["Metadata"]['network'] + inflow_filter = network_filters[network]['inflow_filter'] + outflow_filter = network_filters[network]['outflow_filter'] + start_filter = network_filters[network]['warmup_steps'] + + # update baseline if needed + if table == 'fact_vehicle_trace' \ + and 'is_baseline' in response['Metadata'] and response['Metadata']['is_baseline'] == 'True': + update_baseline(s3, network, source_id) + + readied_queries = get_ready_queries(completed[source_id], query_name) + completed[source_id].add(query_name) + # stores the updated list of completed queries back to S3 + put_completed_queries(s3, source_id, completed[source_id]) + # initialize queries and store them at appropriate locations + for readied_query_name, table_name in readied_queries: + result_location = 's3://circles.data.pipeline/{}/date={}/partition_name={}_{}'.format(table_name, + query_date, + source_id, + readied_query_name) + message_body = (readied_query_name, result_location, query_date, partition, inflow_filter, outflow_filter, + start_filter, max_decel, leader_max_decel) + message_body = json.dumps(message_body) + sqs.send_message( + QueueUrl="", + MessageBody=message_body) diff --git a/flow/data_pipeline/leaderboard_utils.py b/flow/data_pipeline/leaderboard_utils.py new file mode 100644 index 000000000..dd7055f8b --- /dev/null +++ b/flow/data_pipeline/leaderboard_utils.py @@ -0,0 +1,166 @@ +"""APIs for the leader board front end.""" +import os +import boto3 +import pandas as pd +from io import StringIO + + +network_name_map = {"highway": "Single-Lane Straight Road", + "highway_single": "Single-Lane Straight Road", + "ring": "Single-Lane Ring Road", + "I-210_subnetwork": "I-210 without Ramps", + "I_210_subnetwork": "I-210 without Ramps"} + + +def network_name_translate(network_name): + """Translate network name to a human friendly name for the leaderboard.""" + return network_name_map.get(network_name, network_name) + + +def key_to_name(key): + """Return the standard formatted file name from object key.""" + k_list = key.split("/") + date = k_list[1].replace("date=", "") + name = k_list[2].replace("partition_name=", "") + index = name.find("_", 5) + source_id = name + query_name = "" + if index != -1: + source_id = name[0:index] + query_name = "_" + name[index+1:].replace("_", "-") + return "{}_{}{}.csv".format(date, source_id.replace("_", "-"), query_name) + + +def get_table_disk(table_name="fact_vehicle_trace", bucket="circles.data.pipeline"): + """Fetch tables from s3 and store in ./result directory. + + Parameters + ---------- + table_name : str + The name of table to retrieve from S3, the current available tables are: + fact_vehicle_trace + fact_energy_trace + fact_network_throughput_agg + fact_network_inflows_outflows + fact_vehicle_fuel_efficiency_agg + fact_network_metrics_by_distance_agg + fact_network_metrics_by_time_agg + fact_network_fuel_efficiency_agg + leaderboard_chart + leaderboard_chart_agg + Note that leaderboard_chart_agg is a combination of all previous + learderboard_chart entries in one CSV file. It's only used to + avoid burdening the web server with more calculation. The date + and source_id in its name is always going to reflect the latest + leaderboard_chart entry. + bucket : str + the S3 bucket that holds these tables + """ + try: + os.makedirs("result/{}".format(table_name)) + except FileExistsError: + pass + s3 = boto3.client("s3") + response = s3.list_objects_v2(Bucket=bucket) + keys = [e["Key"] for e in response["Contents"] if e["Key"].find(table_name) == 0 and e["Key"][-4:] == ".csv"] + names = [key_to_name(k) for k in keys] + existing_results = os.listdir("./result/{}".format(table_name)) + updated = False + for index in range(len(keys)): + if names[index] not in existing_results: + updated = True + s3.download_file(bucket, keys[index], "./result/{}/{}".format(table_name, names[index])) + if table_name == "leaderboard_chart_agg" and updated: + for p in existing_results: + os.remove("./result/{}/{}".format(table_name, p)) + + +def get_table_memory(table_name="fact_vehicle_trace", bucket="circles.data.pipeline", existing_results=()): + """Fetch tables from s3 and return them as in-memory pandas dataframe objects. + + Parameters + ---------- + bucket: str + the S3 bucket that holds the tables + table_name: str + the name of the name to retrieve from S3, for detail see get_table_disk + existing_results: list + tables that should not be fetched, + the names must follow the convention: + {date}_{source_id(no run number)}_{query_name}.csv + + Returns + ------- + file_list: dict + a dictionary of pandas dataframes, each contains a table from S3 + The dataframs are keyed by their name: {source_id(no run number)}_{query_name}.csv + + """ + s3 = boto3.client("s3") + response = s3.list_objects_v2(Bucket=bucket) + keys = [e["Key"] for e in response["Contents"] if e["Key"].find(table_name) == 0 and e["Key"][-4:] == ".csv"] + names = [key_to_name(k) for k in keys] + results = dict() + for index in range(len(keys)): + if names[index] not in existing_results: + obj = s3.get_object(Bucket=bucket, Key=keys[index])["Body"] + obj_str = obj.read().decode("utf-8") + results[names[index]] = pd.read_csv(StringIO(obj_str)) + return results + + +def get_table_url(table_name="fact_vehicle_trace", bucket="circles.data.pipeline", existing_results=()): + """Fetch tables from s3 and return as urls, requires the bucket to have public access. + + Parameters + ---------- + bucket: str + the S3 bucket that holds the tables + table_name: str + the name of the name to retrieve from S3, for detail see get_table_disk + existing_results: list + tables that should not be fetched, + the names must follow the convention: + {date}_{source_id(no run number)}_{query_name}.csv + + Returns + ------- + file_list: dict + a dictionary of urls, each contains a table from S3 + The urls are keyed by their name: {source_id(no run number)}_{query_name}.csv + + """ + s3 = boto3.client("s3") + response = s3.list_objects_v2(Bucket=bucket) + keys = [e["Key"] for e in response["Contents"] if e["Key"].find(table_name) == 0 and e["Key"][-4:] == ".csv"] + names = [key_to_name(k) for k in keys] + results = dict() + for index in range(len(keys)): + if names[index] not in existing_results: + results[names[index]] = "https://{}.s3.{}.amazonaws.com/{}".format(bucket, "us-west-2", keys[index]) + return results + + +def get_metadata(name, bucket="circles.data.pipeline"): + """Get the metadata by name. + + Parameters + ---------- + name: str + the name of the table whose metadata will be returned + bucket: str + the bucket that hold the table + + Returns + ------- + metadata: dict + a dictionary of all the metadata, there is no guarantee + for which keys are included + """ + s3 = boto3.client("s3") + name_list = name.split('_') + source_id = name_list[1].replace('.csv', "").replace('-', '_') + response = s3.head_object(Bucket=bucket, + Key="fact_vehicle_trace/date={0}/partition_name={1}/{1}.csv".format(name_list[0], + source_id)) + return response["Metadata"] diff --git a/flow/data_pipeline/query.py b/flow/data_pipeline/query.py new file mode 100644 index 000000000..9ba11326c --- /dev/null +++ b/flow/data_pipeline/query.py @@ -0,0 +1,935 @@ +"""stores all the pre-defined query strings.""" +from collections import defaultdict +from enum import Enum + +# tags for different queries +prerequisites = { + "TACOMA_FIT_DENOISED_ACCEL": ( + "fact_energy_trace", {"FACT_VEHICLE_TRACE"} + ), + "PRIUS_FIT_DENOISED_ACCEL": ( + "fact_energy_trace", {"FACT_VEHICLE_TRACE"} + ), + "FACT_SAFETY_METRICS_2D": ( + "fact_safety_metrics", {"FACT_VEHICLE_TRACE"} + ), + "FACT_SAFETY_METRICS_3D": ( + "fact_safety_metrics", {"FACT_VEHICLE_TRACE"} + ), + "FACT_NETWORK_THROUGHPUT_AGG": ( + "fact_network_throughput_agg", {"FACT_VEHICLE_TRACE"} + ), + "FACT_NETWORK_INFLOWS_OUTFLOWS": ( + "fact_network_inflows_outflows", {"FACT_VEHICLE_TRACE"} + ), + "FACT_NETWORK_SPEED": ( + "fact_network_speed", {"FACT_VEHICLE_TRACE"} + ), + "FACT_VEHICLE_COUNTS_BY_TIME": ( + "fact_vehicle_counts_by_time", {"FACT_VEHICLE_TRACE"} + ), + "FACT_VEHICLE_FUEL_EFFICIENCY_AGG": ( + "fact_vehicle_fuel_efficiency_agg", {"FACT_VEHICLE_TRACE", + "TACOMA_FIT_DENOISED_ACCEL", + "PRIUS_FIT_DENOISED_ACCEL"} + ), + "FACT_NETWORK_METRICS_BY_DISTANCE_AGG": ( + "fact_network_metrics_by_distance_agg", {"FACT_VEHICLE_TRACE", + "TACOMA_FIT_DENOISED_ACCEL", + "PRIUS_FIT_DENOISED_ACCEL"} + ), + "FACT_NETWORK_METRICS_BY_TIME_AGG": ( + "fact_network_metrics_by_time_agg", {"FACT_VEHICLE_TRACE", + "TACOMA_FIT_DENOISED_ACCEL", + "PRIUS_FIT_DENOISED_ACCEL"} + ), + "FACT_VEHICLE_FUEL_EFFICIENCY_BINNED": ( + "fact_vehicle_fuel_efficiency_binned", {"FACT_VEHICLE_FUEL_EFFICIENCY_AGG"} + ), + "FACT_NETWORK_FUEL_EFFICIENCY_AGG": ( + "fact_network_fuel_efficiency_agg", {"FACT_VEHICLE_FUEL_EFFICIENCY_AGG"} + ), + "FACT_SAFETY_METRICS_AGG": ( + "fact_safety_metrics_agg", {"FACT_SAFETY_METRICS_3D"} + ), + "FACT_SAFETY_METRICS_BINNED": ( + "fact_safety_metrics_binned", {"FACT_SAFETY_METRICS_3D"} + ), + "LEADERBOARD_CHART": ( + "leaderboard_chart", {"FACT_NETWORK_THROUGHPUT_AGG", + "FACT_NETWORK_SPEED", + "FACT_NETWORK_FUEL_EFFICIENCY_AGG", + "FACT_SAFETY_METRICS_AGG"} + ), + "LEADERBOARD_CHART_AGG": ( + "leaderboard_chart_agg", {"LEADERBOARD_CHART"} + ), + "FACT_TOP_SCORES": ( + "fact_top_scores", {"LEADERBOARD_CHART_AGG"} + ), +} + +triggers = [ + "FACT_VEHICLE_TRACE", + "TACOMA_FIT_DENOISED_ACCEL", + "PRIUS_FIT_DENOISED_ACCEL", + "FACT_VEHICLE_FUEL_EFFICIENCY_AGG", + "FACT_SAFETY_METRICS_3D", + "FACT_NETWORK_THROUGHPUT_AGG", + "FACT_NETWORK_SPEED", + "FACT_NETWORK_FUEL_EFFICIENCY_AGG", + "FACT_SAFETY_METRICS_AGG", + "LEADERBOARD_CHART", + "LEADERBOARD_CHART_AGG" +] + +tables = [ + "fact_vehicle_trace", + "fact_energy_trace", + "fact_vehicle_counts_by_time", + "fact_safety_metrics", + "fact_safety_metrics_agg", + "fact_safety_metrics_binned", + "fact_network_throughput_agg", + "fact_network_inflows_outflows", + "fact_network_speed", + "fact_vehicle_fuel_efficiency_agg", + "fact_vehicle_fuel_efficiency_binned", + "fact_network_metrics_by_distance_agg", + "fact_network_metrics_by_time_agg", + "fact_network_fuel_efficiency_agg", + "leaderboard_chart", + "leaderboard_chart_agg", + "fact_top_scores", + "metadata_table" +] + +summary_tables = ["leaderboard_chart_agg", "fact_top_scores"] + +network_filters = defaultdict(lambda: { + 'inflow_filter': "x > 500", + 'outflow_filter': "x < 2300", + 'warmup_steps': 500 * 3 * 0.4 + }) +network_filters['I-210 without Ramps'] = { + 'inflow_filter': "edge_id != 'ghost0'", + 'outflow_filter': "edge_id != '119257908#3'", + 'warmup_steps': 600 * 3 * 0.4 + } + +max_decel = -1.0 +leader_max_decel = -2.0 + +TACOMA_FIT_FINAL_SELECT = """ + SELECT + id, + time_step, + speed, + acceleration, + road_grade, + GREATEST(0, 2041 * acceleration * speed + + 3405.5481762 + + 83.12392997 * speed + + 6.7650718327 * POW(speed,2) + + 0.7041355229 * POW(speed,3) + ) + GREATEST(0, 4598.7155 * acceleration + 975.12719 * acceleration * speed) AS power, + 'TACOMA_FIT_DENOISED_ACCEL' AS energy_model_id, + source_id + FROM {} + ORDER BY id, time_step + """ + +PRIUS_FIT_FINAL_SELECT = """ + , pmod_calculation AS ( + SELECT + id, + time_step, + speed, + acceleration, + road_grade, + GREATEST(1663 * acceleration * speed + + 1.046 + + 119.166 * speed + + 0.337 * POW(speed,2) + + 0.383 * POW(speed,3) + + GREATEST(0, 296.66 * acceleration * speed)) AS p_mod, + source_id + FROM {} + ) + SELECT + id, + time_step, + speed, + acceleration, + road_grade, + GREATEST(p_mod, 0.869 * p_mod, -2338 * speed) AS power, + 'PRIUS_FIT_DENOISED_ACCEL' AS energy_model_id, + source_id + FROM pmod_calculation + ORDER BY id, time_step + """ + +DENOISED_ACCEL = """ + WITH denoised_accel_cte AS ( + SELECT + id, + time_step, + speed, + COALESCE (target_accel_no_noise_with_failsafe, + target_accel_no_noise_no_failsafe, + realized_accel) AS acceleration, + road_grade, + source_id + FROM fact_vehicle_trace + WHERE 1 = 1 + AND date = \'{{date}}\' + AND partition_name=\'{{partition}}\' + ) + {}""" + + +class QueryStrings(Enum): + """An enumeration of all the pre-defined query strings.""" + + SAMPLE = """ + SELECT * + FROM trajectory_table + WHERE date = \'{date}\' + AND partition_name=\'{partition}\' + LIMIT 15; + """ + + UPDATE_PARTITION = """ + ALTER TABLE {table} + ADD IF NOT EXISTS PARTITION (date = \'{date}\', partition_name=\'{partition}\'); + """ + + TACOMA_FIT_DENOISED_ACCEL = \ + DENOISED_ACCEL.format(TACOMA_FIT_FINAL_SELECT.format('denoised_accel_cte')) + + PRIUS_FIT_DENOISED_ACCEL = \ + DENOISED_ACCEL.format(PRIUS_FIT_FINAL_SELECT.format('denoised_accel_cte')) + + FACT_SAFETY_METRICS_2D = """ + SELECT + vt.id, + vt.time_step, + COALESCE(( + value_lower_left*(headway_upper-headway)*(rel_speed_upper-leader_rel_speed) + + value_lower_right*(headway-headway_lower)*(rel_speed_upper-leader_rel_speed) + + value_upper_left*(headway_upper-headway)*(leader_rel_speed-rel_speed_lower) + + value_upper_right*(headway-headway_lower)*(leader_rel_speed-rel_speed_lower) + ) / ((headway_upper-headway_lower)*(rel_speed_upper-rel_speed_lower)), 200.0) AS safety_value, + 'v2D_HJI' AS safety_model, + vt.source_id + FROM fact_vehicle_trace vt + LEFT OUTER JOIN fact_safety_matrix sm ON 1 = 1 + AND vt.leader_rel_speed BETWEEN sm.rel_speed_lower AND sm.rel_speed_upper + AND vt.headway BETWEEN sm.headway_lower AND sm.headway_upper + WHERE 1 = 1 + AND vt.date = \'{date}\' + AND vt.partition_name = \'{partition}\' + AND vt.time_step >= {start_filter} + AND vt.{inflow_filter} + AND vt.{outflow_filter} + ; + """ + + FACT_SAFETY_METRICS_3D = """ + SELECT + id, + time_step, + headway + (CASE + WHEN -speed/{max_decel} > -(speed+leader_rel_speed)/{leader_max_decel} THEN + -0.5*POW(leader_rel_speed, 2)/{leader_max_decel} + + -0.5*POW(speed,2)/{leader_max_decel} + + -speed*leader_rel_speed/{leader_max_decel} + + 0.5*POW(speed,2)/{max_decel} + ELSE + -leader_rel_speed*speed/{max_decel} + + 0.5*POW(speed,2)*{leader_max_decel}/POW({max_decel},2) + + -0.5*POW(speed,2)/{max_decel} + END) AS safety_value, + 'v3D' AS safety_model, + source_id + FROM fact_vehicle_trace + WHERE 1 = 1 + AND date = \'{date}\' + AND partition_name = \'{partition}\' + AND leader_id IS NOT NULL + AND time_step >= {start_filter} + AND {inflow_filter} + AND {outflow_filter} + ; + """ + + FACT_SAFETY_METRICS_AGG = """ + SELECT + source_id, + SUM(CASE WHEN safety_value > 0 THEN 1.0 ELSE 0.0 END) * 100.0 / COUNT() safety_rate, + MIN(safety_value) AS safety_value_max + FROM fact_safety_metrics + WHERE 1 = 1 + AND date = \'{date}\' + AND partition_name = \'{partition}_FACT_SAFETY_METRICS_3D\' + AND safety_model = 'v3D' + GROUP BY 1 + ; + """ + + FACT_SAFETY_METRICS_BINNED = """ + WITH unfilter_bins AS ( + SELECT + ROW_NUMBER() OVER() - 51 AS lb, + ROW_NUMBER() OVER() - 50 AS ub + FROM fact_safety_matrix + ), bins AS ( + SELECT + lb, + ub + FROM unfilter_bins + WHERE 1=1 + AND lb >= -5 + AND ub <= 15 + ) + SELECT + CONCAT('[', CAST(bins.lb AS VARCHAR), ', ', CAST(bins.ub AS VARCHAR), ')') AS safety_value_bin, + COUNT() AS count + FROM bins + LEFT JOIN fact_safety_metrics fsm ON 1 = 1 + AND fsm.date = \'{date}\' + AND fsm.partition_name = \'{partition}_FACT_SAFETY_METRICS_3D\' + AND fsm.safety_value >= bins.lb + AND fsm.safety_value < bins.ub + AND fsm.safety_model = 'v3D' + GROUP BY 1 + ; + """ + + FACT_NETWORK_THROUGHPUT_AGG = """ + WITH min_time AS ( + SELECT + source_id, + id, + MIN(time_step) AS enter_time + FROM fact_vehicle_trace + WHERE 1 = 1 + AND date = \'{date}\' + AND partition_name = \'{partition}\' + AND {inflow_filter} + GROUP BY 1, 2 + ), agg AS ( + SELECT + source_id, + COUNT(DISTINCT id) AS n_vehicles, + MAX(enter_time) - MIN(enter_time) AS total_time_seconds + FROM min_time + WHERE 1 = 1 + AND enter_time >= {start_filter} + GROUP BY 1 + ) + SELECT + source_id, + n_vehicles * 3600 / total_time_seconds AS throughput_per_hour + FROM agg + ;""" + + FACT_VEHICLE_FUEL_EFFICIENCY_AGG = """ + WITH sub_fact_vehicle_trace AS ( + SELECT + v.id, + v.source_id, + e.energy_model_id, + MAX(distance) - MIN(distance) AS distance_meters, + (MAX(e.time_step) - MIN(e.time_step)) / (COUNT(DISTINCT e.time_step) - 1) AS time_step_size_seconds, + SUM(e.power) AS power_watts + FROM fact_vehicle_trace v + JOIN fact_energy_trace AS e ON 1 = 1 + AND e.id = v.id + AND e.time_step = v.time_step + AND e.source_id = v.source_id + AND e.date = \'{date}\' + AND e.partition_name LIKE \'{partition}_%\' + AND e.time_step >= {start_filter} + WHERE 1 = 1 + AND v.date = \'{date}\' + AND v.partition_name = \'{partition}\' + AND v.{inflow_filter} + AND v.{outflow_filter} + GROUP BY 1, 2, 3 + HAVING 1 = 1 + AND MAX(distance) - MIN(distance) > 10 + AND COUNT(DISTINCT e.time_step) > 10 + ) + SELECT + id, + source_id, + energy_model_id, + distance_meters, + power_watts * time_step_size_seconds AS energy_joules, + distance_meters / (power_watts * time_step_size_seconds) AS efficiency_meters_per_joules, + (CASE energy_model_id + WHEN 'TACOMA_FIT_DENOISED_ACCEL' THEN 33554.13 + WHEN 'PRIUS_FIT_DENOISED_ACCEL' THEN 75384.94 + END) * distance_meters / (power_watts * time_step_size_seconds) AS efficiency_miles_per_gallon + FROM sub_fact_vehicle_trace + WHERE 1 = 1 + AND power_watts * time_step_size_seconds != 0 + ; + """ + + FACT_VEHICLE_FUEL_EFFICIENCY_BINNED = """ + WITH unfilter_bins AS ( + SELECT + ROW_NUMBER() OVER() - 1 AS lb, + ROW_NUMBER() OVER() AS ub + FROM fact_safety_matrix + ), tacoma_binned AS ( + SELECT + bins.lb, + bins.ub, + COUNT() AS count + FROM unfilter_bins bins + LEFT JOIN fact_vehicle_fuel_efficiency_agg agg ON 1 = 1 + AND agg.date = \'{date}\' + AND agg.partition_name = \'{partition}_FACT_VEHICLE_FUEL_EFFICIENCY_AGG\' + AND agg.efficiency_miles_per_gallon >= bins.lb + AND agg.efficiency_miles_per_gallon < bins.ub + AND agg.energy_model_id = 'TACOMA_FIT_DENOISED_ACCEL' + GROUP BY 1, 2 + ), prius_binned AS ( + SELECT + bins.lb, + bins.ub, + COUNT() AS count + FROM unfilter_bins bins + LEFT JOIN fact_vehicle_fuel_efficiency_agg agg ON 1 = 1 + AND agg.date = \'{date}\' + AND agg.partition_name = \'{partition}_FACT_VEHICLE_FUEL_EFFICIENCY_AGG\' + AND agg.efficiency_miles_per_gallon >= bins.lb + AND agg.efficiency_miles_per_gallon < bins.ub + AND agg.energy_model_id = 'PRIUS_FIT_DENOISED_ACCEL' + GROUP BY 1, 2 + ), tacoma_ratio_to_report AS ( + SELECT + lb, + ub, + 100.0 * count / (SUM(count) OVER()) AS count + FROM tacoma_binned + ORDER BY lb + ), prius_ratio_to_report AS ( + SELECT + lb, + ub, + 100.0 * count / (SUM(count) OVER()) AS count + FROM prius_binned + ORDER BY lb + ) + SELECT + 'TACOMA_FIT_DENOISED_ACCEL' AS energy_model_id, + CONCAT('[', CAST(lb AS VARCHAR), ', ', CAST(ub AS VARCHAR), ')') AS fuel_efficiency_bin, + count + FROM tacoma_ratio_to_report + WHERE 1 = 1 + AND lb >= 0 + AND ub <= 60 + UNION ALL + SELECT + 'PRIUS_FIT_DENOISED_ACCEL' AS energy_model_id, + CONCAT('[', CAST(lb AS VARCHAR), ', ', CAST(ub AS VARCHAR), ')') AS fuel_efficiency_bin, + count + FROM prius_ratio_to_report + WHERE 1 = 1 + AND lb >= 130 + AND ub <= 190 + ;""" + + FACT_NETWORK_FUEL_EFFICIENCY_AGG = """ + WITH aggs AS ( + SELECT + source_id, + energy_model_id, + SUM(distance_meters) AS distance_meters, + SUM(energy_joules) AS energy_joules + FROM fact_vehicle_fuel_efficiency_agg + WHERE 1 = 1 + AND date = \'{date}\' + AND partition_name = \'{partition}_FACT_VEHICLE_FUEL_EFFICIENCY_AGG\' + GROUP BY 1, 2 + ) + SELECT + source_id, + energy_model_id, + 1000 * distance_meters / energy_joules AS efficiency_meters_per_kilojoules, + (CASE energy_model_id + WHEN 'TACOMA_FIT_DENOISED_ACCEL' THEN 33554.13 + WHEN 'PRIUS_FIT_DENOISED_ACCEL' THEN 75384.94 + END) * distance_meters / energy_joules AS efficiency_miles_per_gallon + FROM aggs + WHERE 1 = 1 + AND energy_joules != 0 + ;""" + + FACT_NETWORK_SPEED = """ + WITH vehicle_agg AS ( + SELECT + id, + source_id, + AVG(speed) AS vehicle_avg_speed, + COUNT(DISTINCT time_step) AS n_steps, + MAX(time_step) - MIN(time_step) AS time_delta, + MAX(distance) - MIN(distance) AS distance_delta + FROM fact_vehicle_trace + WHERE 1 = 1 + AND date = \'{date}\' + AND partition_name = \'{partition}\' + AND {inflow_filter} + AND {outflow_filter} + AND time_step >= {start_filter} + GROUP BY 1, 2 + ) + SELECT + source_id, + SUM(vehicle_avg_speed * n_steps) / SUM(n_steps) AS avg_instantaneous_speed, + SUM(distance_delta) / SUM(time_delta) AS avg_network_speed + FROM vehicle_agg + GROUP BY 1 + ;""" + + LEADERBOARD_CHART = """ + SELECT + nt.source_id, + nt.throughput_per_hour, + ns.avg_instantaneous_speed, + ns.avg_network_speed, + sm.safety_rate, + sm.safety_value_max, + AVG(CASE + WHEN fe.energy_model_id = 'PRIUS_FIT_DENOISED_ACCEL' + THEN fe.efficiency_meters_per_kilojoules END) AS prius_efficiency_meters_per_kilojoules, + AVG(CASE + WHEN fe.energy_model_id = 'TACOMA_FIT_DENOISED_ACCEL' + THEN fe.efficiency_meters_per_kilojoules END) AS tacoma_efficiency_meters_per_kilojoules, + AVG(CASE + WHEN fe.energy_model_id = 'PRIUS_FIT_DENOISED_ACCEL' + THEN fe.efficiency_miles_per_gallon END) AS prius_efficiency_miles_per_gallon, + AVG(CASE + WHEN fe.energy_model_id = 'TACOMA_FIT_DENOISED_ACCEL' + THEN fe.efficiency_miles_per_gallon END) AS tacoma_efficiency_miles_per_gallon + FROM fact_network_throughput_agg AS nt + JOIN fact_network_speed AS ns ON 1 = 1 + AND ns.date = \'{date}\' + AND ns.partition_name = \'{partition}_FACT_NETWORK_SPEED\' + AND nt.source_id = ns.source_id + JOIN fact_network_fuel_efficiency_agg AS fe ON 1 = 1 + AND fe.date = \'{date}\' + AND fe.partition_name = \'{partition}_FACT_NETWORK_FUEL_EFFICIENCY_AGG\' + AND nt.source_id = fe.source_id + JOIN fact_safety_metrics_agg AS sm ON 1 = 1 + AND sm.date = \'{date}\' + AND sm.partition_name = \'{partition}_FACT_SAFETY_METRICS_AGG\' + AND nt.source_id = sm.source_id + WHERE 1 = 1 + AND nt.date = \'{date}\' + AND nt.partition_name = \'{partition}_FACT_NETWORK_THROUGHPUT_AGG\' + GROUP BY 1, 2, 3, 4, 5, 6 + ;""" + + FACT_NETWORK_INFLOWS_OUTFLOWS = """ + WITH in_out_time_step AS ( + SELECT + id, + source_id, + MIN(CASE WHEN {inflow_filter} THEN time_step - {start_filter} ELSE NULL END) AS inflow_time_step, + MIN(CASE WHEN {outflow_filter} THEN NULL ELSE time_step - {start_filter} END) AS outflow_time_step + FROM fact_vehicle_trace + WHERE 1 = 1 + AND date = \'{date}\' + AND partition_name = \'{partition}\' + GROUP BY 1, 2 + ), inflows AS ( + SELECT + CAST(inflow_time_step / 60 AS INTEGER) * 60 AS time_step, + source_id, + 60 * COUNT(DISTINCT id) AS inflow_rate + FROM in_out_time_step + GROUP BY 1, 2 + ), outflows AS ( + SELECT + CAST(outflow_time_step / 60 AS INTEGER) * 60 AS time_step, + source_id, + 60 * COUNT(DISTINCT id) AS outflow_rate + FROM in_out_time_step + GROUP BY 1, 2 + ) + SELECT + COALESCE(i.time_step, o.time_step) AS time_step, + COALESCE(i.source_id, o.source_id) AS source_id, + COALESCE(i.inflow_rate, 0) AS inflow_rate, + COALESCE(o.outflow_rate, 0) AS outflow_rate + FROM inflows i + FULL OUTER JOIN outflows o ON 1 = 1 + AND i.time_step = o.time_step + AND i.source_id = o.source_id + WHERE 1 = 1 + AND COALESCE(i.time_step, o.time_step) >= 0 + ORDER BY time_step + ;""" + + FACT_NETWORK_METRICS_BY_DISTANCE_AGG = """ + WITH joined_trace AS ( + SELECT + vt.id, + vt.source_id, + vt.time_step, + vt.distance - FIRST_VALUE(vt.distance) + OVER (PARTITION BY vt.id, vt.source_id ORDER BY vt.time_step ASC) AS distance_meters, + energy_model_id, + et.speed, + et.acceleration, + vt.time_step - LAG(vt.time_step, 1) + OVER (PARTITION BY vt.id, vt.source_id ORDER BY vt.time_step ASC) AS sim_step, + SUM(power) + OVER (PARTITION BY vt.id, vt.source_id ORDER BY vt.time_step ASC + ROWS BETWEEN UNBOUNDED PRECEDING and CURRENT ROW) AS cumulative_power + FROM fact_vehicle_trace vt + JOIN fact_energy_trace et ON 1 = 1 + AND et.date = \'{date}\' + AND et.partition_name = \'{partition}_TACOMA_FIT_DENOISED_ACCEL\' + AND vt.id = et.id + AND vt.source_id = et.source_id + AND vt.time_step = et.time_step + AND et.energy_model_id = 'TACOMA_FIT_DENOISED_ACCEL' + WHERE 1 = 1 + AND vt.date = \'{date}\' + AND vt.partition_name = \'{partition}\' + AND vt.{inflow_filter} + AND vt.{outflow_filter} + AND vt.time_step >= {start_filter} + ), cumulative_energy AS ( + SELECT + id, + source_id, + time_step, + distance_meters, + energy_model_id, + speed, + acceleration, + cumulative_power * sim_step AS energy_joules + FROM joined_trace + ), binned_cumulative_energy AS ( + SELECT + source_id, + CAST(distance_meters/10 AS INTEGER) * 10 AS distance_meters_bin, + AVG(speed) AS speed_avg, + AVG(speed) + STDDEV(speed) AS speed_upper_bound, + AVG(speed) - STDDEV(speed) AS speed_lower_bound, + AVG(acceleration) AS accel_avg, + AVG(acceleration) + STDDEV(acceleration) AS accel_upper_bound, + AVG(acceleration) - STDDEV(acceleration) AS accel_lower_bound, + AVG(energy_joules) AS cumulative_energy_avg, + AVG(energy_joules) + STDDEV(energy_joules) AS cumulative_energy_upper_bound, + AVG(energy_joules) - STDDEV(energy_joules) AS cumulative_energy_lower_bound + FROM cumulative_energy + GROUP BY 1, 2 + HAVING 1 = 1 + AND COUNT(DISTINCT time_step) > 1 + ), binned_energy_start_end AS ( + SELECT DISTINCT + source_id, + id, + CAST(distance_meters/10 AS INTEGER) * 10 AS distance_meters_bin, + FIRST_VALUE(energy_joules) + OVER (PARTITION BY id, CAST(distance_meters/10 AS INTEGER) * 10 + ORDER BY time_step ASC) AS energy_start, + LAST_VALUE(energy_joules) + OVER (PARTITION BY id, CAST(distance_meters/10 AS INTEGER) * 10 + ORDER BY time_step ASC) AS energy_end + FROM cumulative_energy + ), binned_energy AS ( + SELECT + source_id, + distance_meters_bin, + AVG(energy_end - energy_start) AS instantaneous_energy_avg, + AVG(energy_end - energy_start) + STDDEV(energy_end - energy_start) AS instantaneous_energy_upper_bound, + AVG(energy_end - energy_start) - STDDEV(energy_end - energy_start) AS instantaneous_energy_lower_bound + FROM binned_energy_start_end + GROUP BY 1, 2 + ) + SELECT + bce.source_id AS source_id, + bce.distance_meters_bin AS distance_meters_bin, + bce.cumulative_energy_avg, + bce.cumulative_energy_lower_bound, + bce.cumulative_energy_upper_bound, + bce.speed_avg, + bce.speed_upper_bound, + bce.speed_lower_bound, + bce.accel_avg, + bce.accel_upper_bound, + bce.accel_lower_bound, + COALESCE(be.instantaneous_energy_avg, 0) AS instantaneous_energy_avg, + COALESCE(be.instantaneous_energy_upper_bound, 0) AS instantaneous_energy_upper_bound, + COALESCE(be.instantaneous_energy_lower_bound, 0) AS instantaneous_energy_lower_bound + FROM binned_cumulative_energy bce + JOIN binned_energy be ON 1 = 1 + AND bce.source_id = be.source_id + AND bce.distance_meters_bin = be.distance_meters_bin + ORDER BY distance_meters_bin ASC + ;""" + + FACT_NETWORK_METRICS_BY_TIME_AGG = """ + WITH joined_trace AS ( + SELECT + vt.id, + vt.source_id, + vt.time_step - FIRST_VALUE(vt.time_step) + OVER (PARTITION BY vt.id, vt.source_id ORDER BY vt.time_step ASC) AS time_step, + energy_model_id, + et.speed, + et.acceleration, + vt.time_step - LAG(vt.time_step, 1) + OVER (PARTITION BY vt.id, vt.source_id ORDER BY vt.time_step ASC) AS sim_step, + SUM(power) + OVER (PARTITION BY vt.id, vt.source_id ORDER BY vt.time_step ASC + ROWS BETWEEN UNBOUNDED PRECEDING and CURRENT ROW) AS cumulative_power + FROM fact_vehicle_trace vt + JOIN fact_energy_trace et ON 1 = 1 + AND et.date = \'{date}\' + AND et.partition_name = \'{partition}_TACOMA_FIT_DENOISED_ACCEL\' + AND vt.id = et.id + AND vt.source_id = et.source_id + AND vt.time_step = et.time_step + AND et.energy_model_id = 'TACOMA_FIT_DENOISED_ACCEL' + WHERE 1 = 1 + AND vt.date = \'{date}\' + AND vt.partition_name = \'{partition}\' + AND vt.{inflow_filter} + AND vt.{outflow_filter} + AND vt.time_step >= {start_filter} + ), cumulative_energy AS ( + SELECT + id, + source_id, + time_step, + energy_model_id, + speed, + acceleration, + cumulative_power * sim_step AS energy_joules + FROM joined_trace + ), binned_cumulative_energy AS ( + SELECT + source_id, + CAST(time_step/10 AS INTEGER) * 10 AS time_seconds_bin, + AVG(speed) AS speed_avg, + AVG(speed) + STDDEV(speed) AS speed_upper_bound, + AVG(speed) - STDDEV(speed) AS speed_lower_bound, + AVG(acceleration) AS accel_avg, + AVG(acceleration) + STDDEV(acceleration) AS accel_upper_bound, + AVG(acceleration) - STDDEV(acceleration) AS accel_lower_bound, + AVG(energy_joules) AS cumulative_energy_avg, + AVG(energy_joules) + STDDEV(energy_joules) AS cumulative_energy_upper_bound, + AVG(energy_joules) - STDDEV(energy_joules) AS cumulative_energy_lower_bound + FROM cumulative_energy + GROUP BY 1, 2 + HAVING 1 = 1 + AND COUNT(DISTINCT time_step) > 1 + ), binned_energy_start_end AS ( + SELECT DISTINCT + source_id, + id, + CAST(time_step/10 AS INTEGER) * 10 AS time_seconds_bin, + FIRST_VALUE(energy_joules) + OVER (PARTITION BY id, CAST(time_step/10 AS INTEGER) * 10 + ORDER BY time_step ASC) AS energy_start, + LAST_VALUE(energy_joules) + OVER (PARTITION BY id, CAST(time_step/10 AS INTEGER) * 10 + ORDER BY time_step ASC) AS energy_end + FROM cumulative_energy + ), binned_energy AS ( + SELECT + source_id, + time_seconds_bin, + AVG(energy_end - energy_start) AS instantaneous_energy_avg, + AVG(energy_end - energy_start) + STDDEV(energy_end - energy_start) AS instantaneous_energy_upper_bound, + AVG(energy_end - energy_start) - STDDEV(energy_end - energy_start) AS instantaneous_energy_lower_bound + FROM binned_energy_start_end + GROUP BY 1, 2 + ) + SELECT + bce.source_id AS source_id, + bce.time_seconds_bin AS time_seconds_bin, + bce.cumulative_energy_avg, + bce.cumulative_energy_lower_bound, + bce.cumulative_energy_upper_bound, + bce.speed_avg, + bce.speed_upper_bound, + bce.speed_lower_bound, + bce.accel_avg, + bce.accel_upper_bound, + bce.accel_lower_bound, + COALESCE(be.instantaneous_energy_avg, 0) AS instantaneous_energy_avg, + COALESCE(be.instantaneous_energy_upper_bound, 0) AS instantaneous_energy_upper_bound, + COALESCE(be.instantaneous_energy_lower_bound, 0) AS instantaneous_energy_lower_bound + FROM binned_cumulative_energy bce + JOIN binned_energy be ON 1 = 1 + AND bce.source_id = be.source_id + AND bce.time_seconds_bin = be.time_seconds_bin + ORDER BY time_seconds_bin ASC + ;""" + + FACT_VEHICLE_COUNTS_BY_TIME = """ + WITH counts AS ( + SELECT + vt.source_id, + vt.time_step, + COUNT(DISTINCT vt.id) AS vehicle_count + FROM fact_vehicle_trace vt + WHERE 1 = 1 + AND vt.date = \'{date}\' + AND vt.partition_name = \'{partition}\' + AND vt.{inflow_filter} + AND vt.{outflow_filter} + AND vt.time_step >= {start_filter} + GROUP BY 1, 2 + ) + SELECT + source_id, + time_step - FIRST_VALUE(time_step) + OVER (PARTITION BY source_id ORDER BY time_step ASC) AS time_step, + vehicle_count + FROM counts + ; + """ + + LEADERBOARD_CHART_AGG = """ + WITH agg AS ( + SELECT + l.date AS submission_date, + m.submission_time, + l.source_id, + m.submitter_name, + m.strategy, + m.network, + m.is_baseline, + COALESCE (m.penetration_rate, 'x') AS penetration_rate, + COALESCE (m.version, '2.0') AS version, + COALESCE (m.road_grade, 'False') AS road_grade, + COALESCE (m.on_ramp, 'False') AS on_ramp, + l.prius_efficiency_meters_per_kilojoules, + l.tacoma_efficiency_meters_per_kilojoules, + l.prius_efficiency_miles_per_gallon, + l.tacoma_efficiency_miles_per_gallon, + l.throughput_per_hour, + l.avg_instantaneous_speed, + l.avg_network_speed, + l.safety_rate, + l.safety_value_max, + b.source_id AS baseline_source_id + FROM leaderboard_chart AS l, metadata_table AS m, baseline_table as b + WHERE 1 = 1 + AND l.source_id = m.source_id + AND m.network = b.network + AND (m.is_baseline='False' + OR (m.is_baseline='True' + AND m.source_id = b.source_id)) + ), joined_cols AS ( + SELECT + agg.submission_date, + agg.source_id, + agg.submitter_name, + agg.strategy, + agg.network || ';' || + ' v' || agg.version || ';' || + ' PR: ' || agg.penetration_rate || '%;' || + CASE agg.on_ramp WHEN + 'True' THEN ' with ramps;' + ELSE ' no ramps;' END || + CASE agg.road_grade WHEN + 'True' THEN ' with grade;' + ELSE ' no grade;' END AS network, + agg.is_baseline, + agg.prius_efficiency_miles_per_gallon, + agg.tacoma_efficiency_miles_per_gallon, + 100 * (1 - + baseline.prius_efficiency_meters_per_kilojoules / agg.prius_efficiency_meters_per_kilojoules) + AS prius_fuel_economy_improvement, + 100 * (1 - + baseline.tacoma_efficiency_meters_per_kilojoules / agg.tacoma_efficiency_meters_per_kilojoules) + AS tacoma_fuel_economy_improvement, + agg.throughput_per_hour, + 100 * (agg.throughput_per_hour - baseline.throughput_per_hour) / baseline.throughput_per_hour + AS throughput_change, + agg.avg_network_speed, + 100 * (agg.avg_network_speed - baseline.avg_network_speed) / baseline.avg_network_speed + AS speed_change, + agg.safety_rate, + agg.safety_value_max + FROM agg + JOIN agg AS baseline ON 1 = 1 + AND agg.network = baseline.network + AND agg.version = baseline.version + AND agg.on_ramp = baseline.on_ramp + AND agg.road_grade = baseline.road_grade + AND baseline.is_baseline = 'True' + AND agg.baseline_source_id = baseline.source_id + ) + SELECT + submission_date, + source_id, + submitter_name, + strategy, + network, + is_baseline, + tacoma_efficiency_miles_per_gallon, + prius_efficiency_miles_per_gallon, + CAST (ROUND(tacoma_efficiency_miles_per_gallon, 1) AS VARCHAR) || + '; ' || CAST (ROUND(prius_efficiency_miles_per_gallon, 1) AS VARCHAR) || + ' (' || (CASE WHEN SIGN(tacoma_fuel_economy_improvement) = 1 THEN '+' ELSE '' END) || + CAST (ROUND(tacoma_fuel_economy_improvement, 1) AS VARCHAR) || '%; ' || + (CASE WHEN SIGN(prius_fuel_economy_improvement) = 1 THEN '+' ELSE '' END) || + CAST (ROUND(prius_fuel_economy_improvement, 1) AS VARCHAR) || '%)' AS efficiency, + CAST (ROUND(throughput_per_hour, 1) AS VARCHAR) || + ' (' || (CASE WHEN SIGN(throughput_change) = 1 THEN '+' ELSE '' END) || + CAST (ROUND(throughput_change, 1) AS VARCHAR) || '%)' AS inflow, + CAST (ROUND(avg_network_speed, 1) AS VARCHAR) || + ' (' || (CASE WHEN SIGN(speed_change) = 1 THEN '+' ELSE '' END) || + CAST (ROUND(speed_change, 1) AS VARCHAR) || '%)' AS speed, + ROUND(safety_rate, 1) AS safety_rate, + ROUND(safety_value_max, 1) AS safety_value_max + FROM joined_cols + ;""" + + FACT_TOP_SCORES = """ + WITH curr_max AS ( + SELECT + network, + submission_date, + MAX(tacoma_efficiency_miles_per_gallon) + OVER (PARTITION BY network ORDER BY submission_date ASC + ROWS BETWEEN UNBOUNDED PRECEDING and CURRENT ROW) AS tacoma_max_score, + MAX(prius_efficiency_miles_per_gallon) + OVER (PARTITION BY network ORDER BY submission_date ASC + ROWS BETWEEN UNBOUNDED PRECEDING and CURRENT ROW) AS prius_max_score + FROM leaderboard_chart_agg + WHERE 1 = 1 + AND is_baseline = 'False' + ), prev_max AS ( + SELECT + network, + submission_date, + LAG(tacoma_max_score, 1) OVER (PARTITION BY network ORDER BY submission_date ASC) AS tacoma_max_score, + LAG(prius_max_score, 1) OVER (PARTITION BY network ORDER BY submission_date ASC) AS prius_max_score + FROM curr_max + ), unioned AS ( + SELECT * FROM curr_max + UNION ALL + SELECT * FROM prev_max + ) + SELECT DISTINCT * + FROM unioned + WHERE 1 = 1 + AND tacoma_max_score IS NOT NULL + AND prius_max_score IS NOT NULL + ORDER BY 1, 2, 3 + ;""" diff --git a/flow/data_pipeline/run_query.py b/flow/data_pipeline/run_query.py new file mode 100644 index 000000000..1eb802205 --- /dev/null +++ b/flow/data_pipeline/run_query.py @@ -0,0 +1,35 @@ +"""runner script for invoking query manually.""" +import argparse +from flow.data_pipeline.data_pipeline import AthenaQuery +from flow.data_pipeline.query import QueryStrings + +parser = argparse.ArgumentParser(prog="run_query", description="runs query on AWS Athena and stores the result to" + "a S3 location") +parser.add_argument("--run", type=str, nargs="+") +parser.add_argument("--result_location", type=str, nargs='?', default="s3://circles.data.pipeline/query-result/") +parser.add_argument("--partition", type=str, nargs='?', default="default") +parser.add_argument("--list_partitions", action="store_true") +parser.add_argument("--check_status", type=str, nargs='+') +parser.add_argument("--list_queries", action="store_true") +parser.add_argument("--test_query", nargs=1) + + +if __name__ == "__main__": + args = parser.parse_args() + queryEngine = AthenaQuery() + + if args.run: + execution_ids = [] + for query_name in args.run: + execution_ids.append(queryEngine.run_query(query_name, args.result_location, partition=args.partition)) + print(execution_ids) + if args.list_partitions: + print(queryEngine.existing_partitions) + if args.check_status: + status = dict() + for execution_id in args.check_status: + status[execution_id] = queryEngine.check_status(execution_id) + print(status) + if args.list_queries: + for q in QueryStrings: + print(q) diff --git a/flow/energy_models/base_energy.py b/flow/energy_models/base_energy.py new file mode 100644 index 000000000..ba5da5080 --- /dev/null +++ b/flow/energy_models/base_energy.py @@ -0,0 +1,62 @@ +"""Script containing the base vehicle energy class.""" +from abc import ABCMeta, abstractmethod + + +class BaseEnergyModel(metaclass=ABCMeta): + """Base energy model class. + + Calculate the instantaneous power consumption of a vehicle in + the network. It returns the power in Watts regardless of the + vehicle type: whether EV or Combustion Engine, Toyota Prius or Tacoma + or non-Toyota vehicles. Non-Toyota vehicles are set by default + to be an averaged-size vehicle. + + Note: road grade is included as an input parameter, but the + functional dependence on road grade is not yet implemented. + """ + + def __init__(self): + # 15 kilowatts = 1 gallon/hour conversion factor + self.conversion = 15e3 + + @abstractmethod + def get_instantaneous_power(self, accel, speed, grade): + """Calculate the instantaneous power consumption of a vehicle. + + Must be implemented by child classes. + + Parameters + ---------- + accel : float + Instantaneous acceleration of the vehicle + speed : float + Instantaneous speed of the vehicle + grade : float + Instantaneous road grade of the vehicle + + Returns + ------- + float + """ + pass + + def get_instantaneous_fuel_consumption(self, accel, speed, grade): + """Calculate the instantaneous fuel consumption of a vehicle. + + Fuel consumption is reported in gallons per hour, with the conversion + rate of 15kW = 1 gallon/hour. + + Parameters + ---------- + accel : float + Instantaneous acceleration of the vehicle + speed : float + Instantaneous speed of the vehicle + grade : float + Instantaneous road grade of the vehicle + + Returns + ------- + float + """ + return self.get_instantaneous_power(accel, speed, grade) / self.conversion diff --git a/flow/energy_models/power_demand.py b/flow/energy_models/power_demand.py new file mode 100644 index 000000000..2fe34d1ed --- /dev/null +++ b/flow/energy_models/power_demand.py @@ -0,0 +1,139 @@ +"""Script containing the vehicle power demand model energy classes.""" +from abc import ABCMeta +import numpy as np + +from flow.energy_models.base_energy import BaseEnergyModel + + +class PowerDemandModel(BaseEnergyModel, metaclass=ABCMeta): + """Vehicle Power Demand base energy model class. + + Calculate power consumption of a vehicle based on physics + derivation. Assumes some vehicle characteristics. The + power calculated here is the lower bound of the actual + power consumed by the vehicle plus a bilinear polynomial + function used as a correction factor. + """ + + def __init__(self, + mass=2041, + idle_coeff=3405.5481762, + linear_friction_coeff=83.123929917, + quadratic_friction_coeff=6.7650718327, + drag_coeff=0.7041355229, + p0_correction=0, + p1_correction=0, + p2_correction=0, + p3_correction=0): + super(PowerDemandModel, self).__init__() + + self.mass = mass + self.phys_power_coeffs = np.array([idle_coeff, + linear_friction_coeff, + quadratic_friction_coeff, + drag_coeff]) + self.power_correction_coeffs = np.array([p0_correction, + p1_correction, + p2_correction, + p3_correction]) + + def calculate_phys_power(self, accel, speed, grade): + """Calculate the instantaneous power from physics-based derivation. + + Parameters + ---------- + accel : float + Instantaneous acceleration of the vehicle + speed : float + Instantaneous speed of the vehicle + grade : float + Instantaneous road grade of the vehicle + Returns + ------- + float + """ + state_variables = np.array([1, speed, speed**2, speed**3]) + power_0 = np.dot(self.phys_power_coeffs, state_variables) + return self.mass * accel * speed + power_0 + + def get_power_correction_factor(self, accel, speed, grade): + """Calculate the instantaneous power correction of a vehicle. + + Parameters + ---------- + accel : float + Instantaneous acceleration of the vehicle + speed : float + Instantaneous speed of the vehicle + grade : float + Instantaneous road grade of the vehicle + + Returns + ------- + float + """ + state_variables = np.array([1, accel, speed, accel * speed]) + return max(0, np.dot(self.power_correction_coeffs, state_variables)) + + def get_instantaneous_power(self, accel, speed, grade): + """See parent class.""" + phys_power = self.calculate_phys_power(accel, speed, grade) + power_correction_factor = self.get_power_correction_factor(accel, speed, grade) + return phys_power + power_correction_factor + + +class PDMCombustionEngine(PowerDemandModel): + """Power Demand Model for a combustion engine vehicle. + + For more information, see docs/Tacoma_EnergyModel.pdf + """ + + def __init__(self, + mass=2041, + idle_coeff=3405.5481762, + linear_friction_coeff=83.123929917, + quadratic_friction_coeff=6.7650718327, + drag_coeff=0.7041355229, + p1_correction=4598.7155, + p3_correction=975.12719): + super(PDMCombustionEngine, self).__init__(mass=mass, + idle_coeff=idle_coeff, + linear_friction_coeff=linear_friction_coeff, + quadratic_friction_coeff=quadratic_friction_coeff, + drag_coeff=drag_coeff, + p1_correction=p1_correction, + p3_correction=p3_correction) + + def calculate_phys_power(self, accel, speed, grade): + """See parent class.""" + return max(super(PDMCombustionEngine, self).calculate_phys_power(accel, speed, grade), 0) + + +class PDMElectric(PowerDemandModel): + """Power Demand Model for an electric vehicle. + + For more information, see docs/Prius_EnergyModel.pdf + """ + + def __init__(self, + mass=1663, + idle_coeff=1.046, + linear_friction_coeff=119.166, + quadratic_friction_coeff=0.337, + drag_coeff=0.383, + p3_correction=296.66, + alpha=0.869, + beta=2338): + super(PDMElectric, self).__init__(mass=mass, + idle_coeff=idle_coeff, + linear_friction_coeff=linear_friction_coeff, + quadratic_friction_coeff=quadratic_friction_coeff, + drag_coeff=drag_coeff, + p3_correction=p3_correction) + self.alpha = alpha + self.beta = beta + + def get_instantaneous_power(self, accel, speed, grade): + """See parent class.""" + mod_power = super(PDMElectric, self).get_instantaneous_power(accel, speed, grade) + return max(mod_power, self.alpha * mod_power, -self.beta * speed) diff --git a/flow/energy_models/toyota_energy.py b/flow/energy_models/toyota_energy.py new file mode 100644 index 000000000..39f9da823 --- /dev/null +++ b/flow/energy_models/toyota_energy.py @@ -0,0 +1,68 @@ +"""Script containing the Toyota energy classes.""" +from abc import ABCMeta, abstractmethod +import dill as pickle +import boto3 +import os + +from flow.energy_models.base_energy import BaseEnergyModel + + +class ToyotaModel(BaseEnergyModel, metaclass=ABCMeta): + """Base Toyota Energy model class.""" + + def __init__(self, filename): + super(ToyotaModel, self).__init__() + + # download file from s3 bucket + s3 = boto3.client('s3') + s3.download_file('toyota.restricted', filename, 'temp.pkl') + + file = open('temp.pkl', 'rb') + try: + self.toyota_energy = pickle.load(file) + # delete pickle file + file.close() + os.remove('temp.pkl') + except TypeError: + print('Must use Python version 3.6.8 to unpickle') + # delete pickle file + file.close() + os.remove('temp.pkl') + raise + + @abstractmethod + def get_instantaneous_power(self, accel, speed, grade): + """See parent class.""" + pass + + +class PriusEnergy(ToyotaModel): + """Toyota Prius (EV) energy model class.""" + + def __init__(self, sim_step, soc=0.9): + super(PriusEnergy, self).__init__(filename='prius_ev.pkl') + self.sim_step = sim_step + self.soc = soc + + def get_instantaneous_power(self, accel, speed, grade): + """See parent class.""" + socdot = self.toyota_energy(self.soc, accel, speed, grade) + self.soc -= socdot * self.sim_step + # FIXME (Joy): convert socdot to power + return socdot + + +class TacomaEnergy(ToyotaModel): + """Toyota Tacoma energy model class.""" + + def __init__(self): + super(TacomaEnergy, self).__init__(filename='tacoma.pkl') + + def get_instantaneous_power(self, accel, speed, grade): + """See parent class.""" + return self.get_instantaneous_fuel_consumption(accel, speed, grade) * self.conversion + + def get_instantaneous_fuel_consumption(self, accel, speed, grade): + """See parent class.""" + fc = self.toyota_energy(accel, speed, grade) + return fc * 3600.0 / 3217.25 diff --git a/flow/envs/__init__.py b/flow/envs/__init__.py index 611ed3d9a..4243fc796 100755 --- a/flow/envs/__init__.py +++ b/flow/envs/__init__.py @@ -11,6 +11,8 @@ from flow.envs.ring.wave_attenuation import WaveAttenuationEnv, \ WaveAttenuationPOEnv from flow.envs.merge import MergePOEnv +from flow.envs.straightroad_env import SingleStraightRoad, \ + SingleStraightRoadEnergyBracketing from flow.envs.test import TestEnv # deprecated classes whose names have changed @@ -37,6 +39,8 @@ 'BottleneckDesiredVelocityEnv', 'TestEnv', 'BayBridgeEnv', + 'SingleStraightRoad', + 'SingleStraightRoadEnergyBracketing', # deprecated classes 'BottleNeckAccelEnv', 'DesiredVelocityEnv', diff --git a/flow/envs/base.py b/flow/envs/base.py index c4462e8c8..a5a1cd52b 100644 --- a/flow/envs/base.py +++ b/flow/envs/base.py @@ -26,6 +26,8 @@ from flow.core.kernel import Kernel from flow.utils.exceptions import FatalFlowError +from flow.data_pipeline.data_pipeline import get_extra_info + class Env(gym.Env, metaclass=ABCMeta): """Base environment class. @@ -149,6 +151,13 @@ def __init__(self, self.state = None self.obs_var_labels = [] + # number of training iterations (used by the rllib training procedure) + self._num_training_iters = 0 + + # track IDs that have ever been observed in the system + self._observed_ids = set() + self._observed_rl_ids = set() + # simulation step size self.sim_step = sim_params.sim_step @@ -323,6 +332,11 @@ def step(self, rl_actions): contains other diagnostic information from the previous action """ for _ in range(self.env_params.sims_per_step): + # This tracks vehicles that have appeared during warmup steps + if self.time_counter <= self.env_params.sims_per_step * self.env_params.warmup_steps: + self._observed_ids.update(self.k.vehicle.get_ids()) + self._observed_rl_ids.update(self.k.vehicle.get_rl_ids()) + self.time_counter += 1 self.step_counter += 1 @@ -397,8 +411,7 @@ def step(self, rl_actions): # test if the environment should terminate due to a collision or the # time horizon being met done = (self.time_counter >= self.env_params.sims_per_step * - (self.env_params.warmup_steps + self.env_params.horizon) - or crash) + (self.env_params.warmup_steps + self.env_params.horizon)) # compute the info for each agent infos = {} @@ -431,6 +444,10 @@ def reset(self): # reset the time counter self.time_counter = 0 + # reset the observed ids + self._observed_ids = set() + self._observed_rl_ids = set() + # Now that we've passed the possibly fake init steps some rl libraries # do, we can feel free to actually render things if self.should_render: @@ -468,55 +485,73 @@ def reset(self): elif self.initial_config.shuffle: self.setup_initial_state() - # clear all vehicles from the network and the vehicles class - if self.simulator == 'traci': - for veh_id in self.k.kernel_api.vehicle.getIDList(): # FIXME: hack + if self.sim_params.load_state is not None: + for veh_id in list(self.k.vehicle.get_ids()): + self.k.vehicle.remove(veh_id, from_sumo=False) + + for veh_id in self.k.kernel_api.vehicle.getIDList(): + self.k.vehicle._add_departed(veh_id, "human") + + # Run assertions to make sure the operation was successful. + assert self.k.vehicle.num_vehicles == \ + len(self.k.kernel_api.vehicle.getIDList()) + assert set(self.k.vehicle.get_ids()) == \ + set(self.k.kernel_api.vehicle.getIDList()), \ + list(set(self.k.vehicle.get_ids()) - + set(self.k.kernel_api.vehicle.getIDList())) + + else: + # clear all vehicles from the network and the vehicles class + if self.simulator == 'traci': + # FIXME: hack + for veh_id in self.k.kernel_api.vehicle.getIDList(): + try: + self.k.vehicle.remove(veh_id) + except (FatalTraCIError, TraCIException): + print(traceback.format_exc()) + + # clear all vehicles from the network and the vehicles class + # FIXME (ev, ak) this is weird and shouldn't be necessary + for veh_id in list(self.k.vehicle.get_ids()): + # do not try to remove the vehicles from the network in the + # first step after initializing the network, as there will be + # no vehicles + if self.step_counter == 0: + continue try: self.k.vehicle.remove(veh_id) except (FatalTraCIError, TraCIException): - print(traceback.format_exc()) - - # clear all vehicles from the network and the vehicles class - # FIXME (ev, ak) this is weird and shouldn't be necessary - for veh_id in list(self.k.vehicle.get_ids()): - # do not try to remove the vehicles from the network in the first - # step after initializing the network, as there will be no vehicles - if self.step_counter == 0: - continue - try: - self.k.vehicle.remove(veh_id) - except (FatalTraCIError, TraCIException): - print("Error during start: {}".format(traceback.format_exc())) + print("Error during start: {}".format(traceback.format_exc())) - # do any additional resetting of the vehicle class needed - self.k.vehicle.reset() + # do any additional resetting of the vehicle class needed + self.k.vehicle.reset() - # reintroduce the initial vehicles to the network - for veh_id in self.initial_ids: - type_id, edge, lane_index, pos, speed = \ - self.initial_state[veh_id] + # reintroduce the initial vehicles to the network + for veh_id in self.initial_ids: + type_id, edge, lane_index, pos, speed = \ + self.initial_state[veh_id] - try: - self.k.vehicle.add( - veh_id=veh_id, - type_id=type_id, - edge=edge, - lane=lane_index, - pos=pos, - speed=speed) - except (FatalTraCIError, TraCIException): - # if a vehicle was not removed in the first attempt, remove it - # now and then reintroduce it - self.k.vehicle.remove(veh_id) - if self.simulator == 'traci': - self.k.kernel_api.vehicle.remove(veh_id) # FIXME: hack - self.k.vehicle.add( - veh_id=veh_id, - type_id=type_id, - edge=edge, - lane=lane_index, - pos=pos, - speed=speed) + try: + self.k.vehicle.add( + veh_id=veh_id, + type_id=type_id, + edge=edge, + lane=lane_index, + pos=pos, + speed=speed) + except (FatalTraCIError, TraCIException): + # if a vehicle was not removed in the first attempt, remove + # it now and then reintroduce it + self.k.vehicle.remove(veh_id) + if self.simulator == 'traci': + self.k.kernel_api.vehicle.remove(veh_id) # FIXME: hack + self.k.vehicle.add( + veh_id=veh_id, + type_id=type_id, + edge=edge, + lane=lane_index, + pos=pos, + speed=speed) # advance the simulation in the simulator by one step self.k.simulation.simulation_step() @@ -554,6 +589,14 @@ def reset(self): # perform (optional) warm-up steps before training for _ in range(self.env_params.warmup_steps): observation, _, _, _ = self.step(rl_actions=None) + # collect data for pipeline during the warmup period + try: + extra_info, source_id, run_id = self.pipeline_params + veh_ids = self.k.vehicle.get_ids() + get_extra_info(self.k.vehicle, extra_info, veh_ids, source_id, run_id) + # In case the attribute `pipeline_params` if not added to this instance + except AttributeError: + pass # render a frame self.render(reset=True) @@ -802,3 +845,7 @@ def pyglet_render(self): sight = self.renderer.get_sight( orientation, id) self.sights.append(sight) + + def set_iteration_num(self): + """Increment the number of training iterations.""" + self._num_training_iters += 1 diff --git a/flow/envs/multiagent/__init__.py b/flow/envs/multiagent/__init__.py index f7889591d..02f45526a 100644 --- a/flow/envs/multiagent/__init__.py +++ b/flow/envs/multiagent/__init__.py @@ -10,7 +10,9 @@ from flow.envs.multiagent.traffic_light_grid import MultiTrafficLightGridPOEnv from flow.envs.multiagent.highway import MultiAgentHighwayPOEnv from flow.envs.multiagent.merge import MultiAgentMergePOEnv -from flow.envs.multiagent.i210 import I210MultiEnv +from flow.envs.multiagent.i210 import I210MultiEnv, MultiStraightRoad, \ + I210TestEnv, I210TransferEnv, StraightRoadTestEnv + __all__ = [ 'MultiEnv', @@ -21,5 +23,9 @@ 'MultiAgentAccelPOEnv', 'MultiAgentWaveAttenuationPOEnv', 'MultiAgentMergePOEnv', - 'I210MultiEnv' + 'I210MultiEnv', + 'I210TestEnv', + 'MultiStraightRoad', + 'StraightRoadTestEnv', + 'I210TransferEnv' ] diff --git a/flow/envs/multiagent/base.py b/flow/envs/multiagent/base.py index 2d9c3cd78..b0f51734d 100644 --- a/flow/envs/multiagent/base.py +++ b/flow/envs/multiagent/base.py @@ -49,6 +49,10 @@ def step(self, rl_actions): contains other diagnostic information from the previous action """ for _ in range(self.env_params.sims_per_step): + if self.time_counter <= self.env_params.sims_per_step * self.env_params.warmup_steps: + self._observed_ids.update(self.k.vehicle.get_ids()) + self._observed_rl_ids.update(self.k.vehicle.get_rl_ids()) + self.time_counter += 1 self.step_counter += 1 @@ -103,6 +107,7 @@ def step(self, rl_actions): # stop collecting new simulation steps if there is a collision if crash: + print('A CRASH! A CRASH!!!!!! AAAAAAAAAH!!!!!') break states = self.get_state() @@ -122,11 +127,11 @@ def step(self, rl_actions): else: reward = self.compute_reward(rl_actions, fail=crash) - for rl_id in self.k.vehicle.get_arrived_rl_ids(self.env_params.sims_per_step): - done[rl_id] = True - reward[rl_id] = 0 - states[rl_id] = np.zeros(self.observation_space.shape[0]) - + if self.env_params.done_at_exit: + for rl_id in self.k.vehicle.get_arrived_rl_ids(self.env_params.sims_per_step): + done[rl_id] = True + reward[rl_id] = 0 + states[rl_id] = -1 * np.ones(self.observation_space.shape[0]) return states, reward, done, infos def reset(self, new_inflow_rate=None): @@ -148,12 +153,17 @@ def reset(self, new_inflow_rate=None): # reset the time counter self.time_counter = 0 + # reset the observed ids + self._observed_ids = set() + self._observed_rl_ids = set() + # Now that we've passed the possibly fake init steps some rl libraries # do, we can feel free to actually render things if self.should_render: self.sim_params.render = True # got to restart the simulation to make it actually display anything self.restart_simulation(self.sim_params) + self.should_render = False # warn about not using restart_instance when using inflows if len(self.net_params.inflows.get()) > 0 and \ @@ -185,55 +195,72 @@ def reset(self, new_inflow_rate=None): elif self.initial_config.shuffle: self.setup_initial_state() - # clear all vehicles from the network and the vehicles class - if self.simulator == 'traci': - for veh_id in self.k.kernel_api.vehicle.getIDList(): # FIXME: hack + if self.sim_params.load_state is not None: + for veh_id in list(self.k.vehicle.get_ids()): + self.k.vehicle.remove(veh_id, from_sumo=False) + + for veh_id in self.k.kernel_api.vehicle.getIDList(): + self.k.vehicle._add_departed(veh_id, "human") + + # Run assertions to make sure the operation was successful. + assert self.k.vehicle.num_vehicles == \ + len(self.k.kernel_api.vehicle.getIDList()) + assert set(self.k.vehicle.get_ids()) == \ + set(self.k.kernel_api.vehicle.getIDList()), \ + list(set(self.k.vehicle.get_ids()) - + set(self.k.kernel_api.vehicle.getIDList())) + else: + # clear all vehicles from the network and the vehicles class + if self.simulator == 'traci': + # FIXME: hack + for veh_id in self.k.kernel_api.vehicle.getIDList(): + try: + self.k.vehicle.remove(veh_id) + except (FatalTraCIError, TraCIException): + print(traceback.format_exc()) + + # clear all vehicles from the network and the vehicles class + # FIXME (ev, ak) this is weird and shouldn't be necessary + for veh_id in list(self.k.vehicle.get_ids()): + # do not try to remove the vehicles from the network in the + # first step after initializing the network, as there will be + # no vehicles + if self.step_counter == 0: + continue try: self.k.vehicle.remove(veh_id) except (FatalTraCIError, TraCIException): - print(traceback.format_exc()) - - # clear all vehicles from the network and the vehicles class - # FIXME (ev, ak) this is weird and shouldn't be necessary - for veh_id in list(self.k.vehicle.get_ids()): - # do not try to remove the vehicles from the network in the first - # step after initializing the network, as there will be no vehicles - if self.step_counter == 0: - continue - try: - self.k.vehicle.remove(veh_id) - except (FatalTraCIError, TraCIException): - print("Error during start: {}".format(traceback.format_exc())) - - # do any additional resetting of the vehicle class needed - self.k.vehicle.reset() - - # reintroduce the initial vehicles to the network - for veh_id in self.initial_ids: - type_id, edge, lane_index, pos, speed = \ - self.initial_state[veh_id] - - try: - self.k.vehicle.add( - veh_id=veh_id, - type_id=type_id, - edge=edge, - lane=lane_index, - pos=pos, - speed=speed) - except (FatalTraCIError, TraCIException): - # if a vehicle was not removed in the first attempt, remove it - # now and then reintroduce it - self.k.vehicle.remove(veh_id) - if self.simulator == 'traci': - self.k.kernel_api.vehicle.remove(veh_id) # FIXME: hack - self.k.vehicle.add( - veh_id=veh_id, - type_id=type_id, - edge=edge, - lane=lane_index, - pos=pos, - speed=speed) + print("Error during start: {}".format(traceback.format_exc())) + + # do any additional resetting of the vehicle class needed + self.k.vehicle.reset() + + # reintroduce the initial vehicles to the network + for veh_id in self.initial_ids: + type_id, edge, lane_index, pos, speed = \ + self.initial_state[veh_id] + + try: + self.k.vehicle.add( + veh_id=veh_id, + type_id=type_id, + edge=edge, + lane=lane_index, + pos=pos, + speed=speed) + except (FatalTraCIError, TraCIException): + # if a vehicle was not removed in the first attempt, remove + # it now and then reintroduce it + self.k.vehicle.remove(veh_id) + if self.simulator == 'traci': + self.k.kernel_api.vehicle.remove(veh_id) # FIXME: hack + self.k.vehicle.add( + veh_id=veh_id, + type_id=type_id, + edge=edge, + lane=lane_index, + pos=pos, + speed=speed) # advance the simulation in the simulator by one step self.k.simulation.simulation_step() diff --git a/flow/envs/multiagent/i210.py b/flow/envs/multiagent/i210.py index 409aeb14f..4cf03c837 100644 --- a/flow/envs/multiagent/i210.py +++ b/flow/envs/multiagent/i210.py @@ -3,19 +3,27 @@ from gym.spaces import Box import numpy as np -from flow.core.rewards import average_velocity +from flow.core.rewards import instantaneous_mpg from flow.envs.multiagent.base import MultiEnv # largest number of lanes on any given edge in the network MAX_LANES = 6 +SPEED_SCALE = 50 +HEADWAY_SCALE = 1000 ADDITIONAL_ENV_PARAMS = { # maximum acceleration for autonomous vehicles, in m/s^2 "max_accel": 1, # maximum deceleration for autonomous vehicles, in m/s^2 "max_decel": 1, - # whether we use an obs space that contains adjacent lane info or just the lead obs + # whether we use an obs space that contains adjacent lane info or just the + # lead obs "lead_obs": True, + # whether the reward should come from local vehicles instead of global + # rewards + "local_reward": True, + # desired velocity + "target_velocity": 25 } @@ -59,6 +67,50 @@ class I210MultiEnv(MultiEnv): def __init__(self, env_params, sim_params, network, simulator='traci'): super().__init__(env_params, sim_params, network, simulator) self.lead_obs = env_params.additional_params.get("lead_obs") + self.reroute_on_exit = env_params.additional_params.get("reroute_on_exit") + self.max_lanes = MAX_LANES + self.num_enter_lanes = 5 + self.entrance_edge = "ghost0" + self.exit_edge = "119257908#3" + self.control_range = env_params.additional_params.get('control_range', None) + self.no_control_edges = env_params.additional_params.get('no_control_edges', []) + self.mpg_reward = env_params.additional_params["mpg_reward"] + self.look_back_length = env_params.additional_params["look_back_length"] + + # dynamics controller for uncontrolled RL vehicles (mimics humans) + controller = self.k.vehicle.type_parameters["human"][ + "acceleration_controller"] + self._human_controller = controller[0]( + veh_id="av", + car_following_params=self.k.vehicle.type_parameters["human"][ + "car_following_params"], + **controller[1] + ) + + # list of all RL vehicles (even out of network) after reroute_on_exit starts + self.reroute_rl_ids = set() + + # whether to add a slight reward for opening up a gap that will be annealed out N iterations in + self.headway_curriculum = env_params.additional_params["headway_curriculum"] + # how many timesteps to anneal the headway curriculum over + self.headway_curriculum_iters = env_params.additional_params["headway_curriculum_iters"] + self.headway_reward_gain = env_params.additional_params["headway_reward_gain"] + self.min_time_headway = env_params.additional_params["min_time_headway"] + + # whether to add a slight reward for opening up a gap that will be annealed out N iterations in + self.speed_curriculum = env_params.additional_params["speed_curriculum"] + # how many timesteps to anneal the headway curriculum over + self.speed_curriculum_iters = env_params.additional_params["speed_curriculum_iters"] + self.speed_reward_gain = env_params.additional_params["speed_reward_gain"] + self.leader = [] + + # penalize stops + self.penalize_stops = env_params.additional_params["penalize_stops"] + self.stop_penalty = env_params.additional_params["stop_penalty"] + + # penalize accel + self.penalize_accel = env_params.additional_params.get("penalize_accel", False) + self.accel_penalty = env_params.additional_params["accel_penalty"] @property def observation_space(self): @@ -74,8 +126,8 @@ def observation_space(self): # speed, dist to ego vehicle, binary value which is 1 if the vehicle is # an AV else: - leading_obs = 3 * MAX_LANES - follow_obs = 3 * MAX_LANES + leading_obs = 3 * self.max_lanes + follow_obs = 3 * self.max_lanes # speed and lane self_obs = 2 @@ -100,89 +152,234 @@ def _apply_rl_actions(self, rl_actions): """See class definition.""" # in the warmup steps, rl_actions is None if rl_actions: + rl_ids = [] + accels = [] for rl_id, actions in rl_actions.items(): accel = actions[0] + controller = self.k.vehicle.get_acc_controller(rl_id) + accel = controller.compute_failsafe(accel, self) + accels.append(accel) + rl_ids.append(rl_id) - # lane_change_softmax = np.exp(actions[1:4]) - # lane_change_softmax /= np.sum(lane_change_softmax) - # lane_change_action = np.random.choice([-1, 0, 1], - # p=lane_change_softmax) + # prevent the AV from blocking the entrance + self.k.vehicle.apply_acceleration(rl_ids, accels) - self.k.vehicle.apply_acceleration(rl_id, accel) - # self.k.vehicle.apply_lane_change(rl_id, lane_change_action) + def in_control_range(self, veh_id): + """Return if a veh_id is on an edge that is allowed to be controlled. + + If control range is defined it uses control range, otherwise it searches over a set of edges + """ + return (self.control_range and self.control_range[1] > + self.k.vehicle.get_x_by_id(veh_id) > self.control_range[0]) or \ + (len(self.no_control_edges) > 0 and self.k.vehicle.get_edge(veh_id) not in + self.no_control_edges) def get_state(self): """See class definition.""" + valid_ids = [rl_id for rl_id in self.k.vehicle.get_rl_ids() if self.in_control_range(rl_id)] if self.lead_obs: veh_info = {} - for rl_id in self.k.vehicle.get_rl_ids(): + for rl_id in valid_ids: speed = self.k.vehicle.get_speed(rl_id) - headway = self.k.vehicle.get_headway(rl_id) - lead_speed = self.k.vehicle.get_speed(self.k.vehicle.get_leader(rl_id)) - if lead_speed == -1001: - lead_speed = 0 - veh_info.update({rl_id: np.array([speed / 50.0, headway / 1000.0, lead_speed / 50.0])}) + lead_id = self.k.vehicle.get_leader(rl_id) + if lead_id in ["", None]: + # in case leader is not visible + lead_speed = SPEED_SCALE + headway = HEADWAY_SCALE + else: + lead_speed = self.k.vehicle.get_speed(lead_id) + headway = self.k.vehicle.get_headway(rl_id) + veh_info.update({rl_id: np.array([speed / SPEED_SCALE, headway / HEADWAY_SCALE, + lead_speed / SPEED_SCALE])}) else: veh_info = {rl_id: np.concatenate((self.state_util(rl_id), self.veh_statistics(rl_id))) - for rl_id in self.k.vehicle.get_rl_ids()} + for rl_id in valid_ids} return veh_info def compute_reward(self, rl_actions, **kwargs): - # TODO(@evinitsky) we need something way better than this. Something that adds - # in notions of local reward """See class definition.""" # in the warmup steps if rl_actions is None: return {} rewards = {} - for rl_id in self.k.vehicle.get_rl_ids(): - if self.env_params.evaluate: - # reward is speed of vehicle if we are in evaluation mode - reward = self.k.vehicle.get_speed(rl_id) - elif kwargs['fail']: - # reward is 0 if a collision occurred - reward = 0 - else: - # reward high system-level velocities - cost1 = average_velocity(self, fail=kwargs['fail']) - - # penalize small time headways - cost2 = 0 - t_min = 1 # smallest acceptable time headway + valid_ids = [rl_id for rl_id in self.k.vehicle.get_rl_ids() if self.in_control_range(rl_id)] + valid_human_ids = [veh_id for veh_id in self.k.vehicle.get_ids() if self.in_control_range(veh_id)] + + if self.env_params.additional_params["local_reward"]: + des_speed = self.env_params.additional_params["target_velocity"] + for rl_id in valid_ids: + rewards[rl_id] = 0 + if self.mpg_reward: + rewards[rl_id] = instantaneous_mpg(self, rl_id, gain=1.0) / 100.0 + follow_id = rl_id + for i in range(self.look_back_length): + follow_id = self.k.vehicle.get_follower(follow_id) + if follow_id not in ["", None]: + rewards[rl_id] += instantaneous_mpg(self, follow_id, gain=1.0) / 100.0 + else: + break + else: + follow_id = rl_id + for i in range(self.look_back_length + 1): + if follow_id not in ["", None]: + follow_speed = self.k.vehicle.get_speed(self.k.vehicle.get_follower(follow_id)) + reward = (des_speed - min(np.abs(follow_speed - des_speed), des_speed)) ** 2 + reward /= ((des_speed ** 2) * self.look_back_length) + rewards[rl_id] += reward + else: + break + follow_id = self.k.vehicle.get_follower(follow_id) - lead_id = self.k.vehicle.get_leader(rl_id) + else: + if self.mpg_reward: + reward = np.nan_to_num(instantaneous_mpg(self, valid_human_ids, gain=1.0)) / 100.0 + else: + speeds = self.k.vehicle.get_speed(valid_human_ids) + des_speed = self.env_params.additional_params["target_velocity"] + # rescale so the critic can estimate it quickly + if self.reroute_on_exit: + reward = np.nan_to_num(np.mean([(des_speed - np.abs(speed - des_speed)) + for speed in speeds]) / des_speed) + else: + reward = np.nan_to_num(np.mean([(des_speed - np.abs(speed - des_speed)) ** 2 + for speed in speeds]) / (des_speed ** 2)) + rewards = {rl_id: reward for rl_id in valid_ids} + + # curriculum over time-gaps + if self.headway_curriculum and self._num_training_iters <= self.headway_curriculum_iters: + t_min = self.min_time_headway # smallest acceptable time headway + for veh_id, rew in rewards.items(): + lead_id = self.k.vehicle.get_leader(veh_id) + penalty = 0 if lead_id not in ["", None] \ - and self.k.vehicle.get_speed(rl_id) > 0: + and self.k.vehicle.get_speed(veh_id) > 0: t_headway = max( - self.k.vehicle.get_headway(rl_id) / - self.k.vehicle.get_speed(rl_id), 0) - cost2 += min((t_headway - t_min) / t_min, 0) - - # weights for cost1, cost2, and cost3, respectively - eta1, eta2 = 1.00, 0.10 - - reward = max(eta1 * cost1 + eta2 * cost2, 0) - - rewards[rl_id] = reward + self.k.vehicle.get_headway(veh_id) / + self.k.vehicle.get_speed(veh_id), 0) + scaling_factor = max(0, 1 - self._num_training_iters / self.headway_curriculum_iters) + penalty += scaling_factor * self.headway_reward_gain * min((t_headway - t_min) / t_min, 0) + + rewards[veh_id] += penalty + + if self.speed_curriculum and self._num_training_iters <= self.speed_curriculum_iters: + des_speed = self.env_params.additional_params["target_velocity"] + + for veh_id, rew in rewards.items(): + speed = self.k.vehicle.get_speed(veh_id) + speed_reward = 0.0 + follow_id = veh_id + for i in range(self.look_back_length): + follow_id = self.k.vehicle.get_follower(follow_id) + if follow_id not in ["", None]: + if self.reroute_on_exit: + speed_reward += (des_speed - np.abs(speed - des_speed)) / des_speed + else: + speed_reward += ((des_speed - np.abs(speed - des_speed)) ** 2) / (des_speed ** 2) + else: + break + scaling_factor = max(0, 1 - self._num_training_iters / self.speed_curriculum_iters) + + rewards[veh_id] += speed_reward * scaling_factor * self.speed_reward_gain + + for veh_id in rewards.keys(): + speed = self.k.vehicle.get_speed(veh_id) + if self.penalize_stops: + if speed < 1.0: + rewards[veh_id] -= self.stop_penalty + if self.penalize_accel and veh_id in self.k.vehicle.previous_speeds: + prev_speed = self.k.vehicle.get_previous_speed(veh_id) + abs_accel = abs(speed - prev_speed) / self.sim_step + rewards[veh_id] -= abs_accel * self.accel_penalty + + # print('time to get reward is ', time() - t) return rewards def additional_command(self): """See parent class. - Define which vehicles are observed for visualization purposes. + Define which vehicles are observed for visualization purposes. Additionally, optionally reroute vehicles + back once they have exited. """ + # In the warmup period all vehicles act as humans. + if self.time_counter < \ + self.env_params.warmup_steps * self.env_params.sims_per_step: + uncontrolled_veh_ids = [veh_id for veh_id in self.k.vehicle.get_ids() if + "human" not in self.k.vehicle.get_type(veh_id)] + + # If no control range is specified, all vehicles are controlled. + elif len(self.no_control_edges) == 0: + uncontrolled_veh_ids = [] + + # Vehicles in the control_range are controlled, others act as humans. + else: + uncontrolled_veh_ids = [] + for veh_id in self.k.vehicle.get_ids(): + if not self.in_control_range(veh_id) and "human" not in self.k.vehicle.get_type(veh_id): + uncontrolled_veh_ids.append(veh_id) + + # Assign accelerations to uncontrolled vehicles. + for veh_id in uncontrolled_veh_ids: + self._human_controller.veh_id = veh_id + acceleration = self._human_controller.get_action(self) + self.k.vehicle.apply_acceleration(veh_id, acceleration) + + super().additional_command() # specify observed vehicles for rl_id in self.k.vehicle.get_rl_ids(): # leader lead_id = self.k.vehicle.get_leader(rl_id) if lead_id: self.k.vehicle.set_observed(lead_id) - # follower - follow_id = self.k.vehicle.get_follower(rl_id) - if follow_id: - self.k.vehicle.set_observed(follow_id) + + if self.reroute_on_exit and self.time_counter >= self.env_params.sims_per_step * self.env_params.warmup_steps \ + and not self.env_params.evaluate: + veh_ids = list(self.k.vehicle.get_ids()) + edges = self.k.vehicle.get_edge(veh_ids) + valid_lanes = list(range(self.num_enter_lanes)) + for veh_id, edge in zip(veh_ids, edges): + if edge == "": + continue + if edge[0] == ":": # center edge + continue + # on the exit edge, near the end, and is the vehicle furthest along + if edge == self.exit_edge and \ + (self.k.vehicle.get_position(veh_id) > self.k.network.edge_length(self.exit_edge) - 100) \ + and self.k.vehicle.get_leader(veh_id) is None: + type_id = self.k.vehicle.get_type(veh_id) + # remove the vehicle + self.k.vehicle.remove(veh_id) + index = np.random.randint(low=0, high=len(valid_lanes)) + lane = valid_lanes[index] + del valid_lanes[index] + # reintroduce it at the start of the network + # Note, the position is 20 so you are not overlapping with the inflow car that is being removed. + # this allows the vehicle to be immediately inserted. + try: + self.k.vehicle.add( + veh_id=veh_id, + edge=self.entrance_edge, + type_id=str(type_id), + lane=str(lane), + pos="20.0", + speed="23.0") + except Exception as e: + print(e) + if len(valid_lanes) == 0: + break + + departed_ids = list(self.k.vehicle.get_departed_ids()) + if isinstance(departed_ids, tuple) and len(departed_ids) > 0: + for veh_id in departed_ids: + if veh_id not in self._observed_ids: + self.k.vehicle.remove(veh_id) + + # update set of all reroute RL vehicles + self.reroute_rl_ids = set(self.k.vehicle.get_rl_ids()) | self.reroute_rl_ids + else: + # reset + self.reroute_rl_ids = set() def state_util(self, rl_id): """Return an array of headway, tailway, leader speed, follower speed. @@ -223,3 +420,136 @@ def veh_statistics(self, rl_id): speed = self.k.vehicle.get_speed(rl_id) / 100.0 lane = (self.k.vehicle.get_lane(rl_id) + 1) / 10.0 return np.array([speed, lane]) + + def step(self, rl_actions): + """See parent class for more details; add option to reroute vehicles.""" + state, reward, done, info = super().step(rl_actions) + if done['__all__']: + # handle the edge case where a vehicle hasn't been put back when the rollout terminates + if self.reroute_on_exit: + for rl_id in self.reroute_rl_ids: + if rl_id not in state.keys(): + done[rl_id] = True + reward[rl_id] = 0 + state[rl_id] = -1 * np.ones(self.observation_space.shape[0]) + # you have to catch the vehicles on the exit edge, they have not yet + # recieved a done when the env terminates + on_exit_edge = [rl_id for rl_id in self.k.vehicle.get_rl_ids() + if self.k.vehicle.get_edge(rl_id) == self.exit_edge] + for rl_id in on_exit_edge: + done[rl_id] = True + reward[rl_id] = 0 + state[rl_id] = -1 * np.ones(self.observation_space.shape[0]) + + return state, reward, done, info + + def reset(self, new_inflow_rate=None): + """Reset the environment.""" + state = super().reset(new_inflow_rate=new_inflow_rate) + # update the network to set the downstream edge speed + if self.env_params.additional_params["randomize_downstream_speed"]: + min_speed = self.env_params.additional_params["min_downstream_speed"] + max_speed = self.env_params.additional_params["max_downstream_speed"] + downstream_speed = np.random.uniform(low=min_speed, high=max_speed) + self.k.network.set_max_speed(self.exit_edge, downstream_speed) + + return state + + +class I210TestEnv(I210MultiEnv): + # TODO(@evinitsky) clean this up, this shouldn't actually subclass MultiEnv + """Version of I210 that overrides non-essential methods to speed up run-time for the non-RL case.""" + + @property + def action_space(self): + """See parent class.""" + return Box(low=0, high=0, shape=(0,), dtype=np.float32) + + @property + def observation_space(self): + """See parent class.""" + return Box(low=0, high=0, shape=(0,), dtype=np.float32) + + def _apply_rl_actions(self, rl_actions): + return + + def compute_reward(self, rl_actions, **kwargs): + """See parent class.""" + return 0 + + def get_state(self, **kwargs): + """See class definition.""" + return {'fake': np.array([])} + + +class I210TransferEnv(I210MultiEnv): + """Env used for transfer tests. + + Almost virtually the same, but with an additional_command method that allows further + flexibility, in particular with outflow speed, which must be called after reset. + """ + + def __init__(self, env_params, sim_params, network, simulator, outflow_speed_limit=None): + super().__init__(env_params, sim_params, network, simulator) + self.outflow_speed_limit = \ + env_params.additional_params.get('outflow_speed_limit', + self.env_params.additional_params.get("max_downstream_speed")) + + def reset(self, new_inflow_rate=None): + """Reset the environment.""" + state = super().reset(new_inflow_rate=new_inflow_rate) + # Update the outflow rate + self.k.kernel_api.edge.setMaxSpeed("119257908#3", self.outflow_speed_limit) + return state + + +class MultiStraightRoad(I210MultiEnv): + """Partially observable multi-agent environment for a straight road. Look at superclass for more information.""" + + def __init__(self, env_params, sim_params, network, simulator): + super().__init__(env_params, sim_params, network, simulator) + self.num_enter_lanes = 1 + self.entrance_edge = self.network.routes['highway_0'][0][0][0] + self.exit_edge = self.network.routes['highway_0'][0][0][-1] + + def _apply_rl_actions(self, rl_actions): + """See class definition.""" + # in the warmup steps, rl_actions is None + if rl_actions: + rl_ids = [] + accels = [] + for rl_id, actions in rl_actions.items(): + accel = actions[0] + controller = self.k.vehicle.get_acc_controller(rl_id) + accel = controller.compute_failsafe(accel, self) + accels.append(accel) + rl_ids.append(rl_id) + + # prevent the AV from blocking the entrance + self.k.vehicle.apply_acceleration(rl_ids, accels) + + +class StraightRoadTestEnv(MultiStraightRoad): + # TODO(@evinitsky) clean this up, this shouldn't actually subclass MultiEnv + """Version of I210 that overrides non-essential methods to speed up run-time for the non-RL case.""" + + @property + def action_space(self): + """See parent class.""" + return Box(low=0, high=0, shape=(0,), dtype=np.float32) + + @property + def observation_space(self): + """See parent class.""" + return Box(low=0, high=0, shape=(0,), dtype=np.float32) + + def _apply_rl_actions(self, rl_actions): + return + + def compute_reward(self, rl_actions, **kwargs): + """See parent class.""" + return 0 + + def get_state(self, **kwargs): + """See class definition.""" + return {'fake': np.array([])} diff --git a/flow/envs/straightroad_env.py b/flow/envs/straightroad_env.py new file mode 100644 index 000000000..1b3e6b824 --- /dev/null +++ b/flow/envs/straightroad_env.py @@ -0,0 +1,285 @@ +"""Environment for training vehicles to reduce congestion in the I210.""" + +from gym.spaces import Box +import numpy as np + +from flow.envs.base import Env + +# largest number of lanes on any given edge in the network +MAX_LANES = 6 +MAX_NUM_VEHS = 8 +SPEED_SCALE = 50 +HEADWAY_SCALE = 1000 + +ADDITIONAL_ENV_PARAMS = { + # maximum acceleration for autonomous vehicles, in m/s^2 + "max_accel": 1, + # maximum deceleration for autonomous vehicles, in m/s^2 + "max_decel": 1, + # whether we use an obs space that contains adjacent lane info or just the lead obs + "lead_obs": True, + # whether the reward should come from local vehicles instead of global rewards + "local_reward": True, + # if the environment terminates once a wave has occurred + "terminate_on_wave": False, + # the environment is not allowed to terminate below this horizon length + 'wave_termination_horizon': 500, + # the speed below which we consider a wave to have occured + 'wave_termination_speed': 10.0 +} + + +class I210SingleEnv(Env): + """Partially observable single-agent environment for the I-210 subnetworks. + + The policy is shared among the agents, so there can be a non-constant + number of RL vehicles throughout the simulation. + Required from env_params: + * max_accel: maximum acceleration for autonomous vehicles, in m/s^2 + * max_decel: maximum deceleration for autonomous vehicles, in m/s^2 + The following states, actions and rewards are considered for one autonomous + vehicle only, as they will be computed in the same way for each of them. + States + The observation consists of the speeds and bumper-to-bumper headways of + the vehicles immediately preceding and following autonomous vehicles in + all of the preceding lanes as well, a binary value indicating which of + these vehicles is autonomous, and the speed of the autonomous vehicle. + Missing vehicles are padded with zeros. + Actions + The action consists of an acceleration, bound according to the + environment parameters, as well as three values that will be converted + into probabilities via softmax to decide of a lane change (left, none + or right). NOTE: lane changing is currently not enabled. It's a TODO. + Rewards + The reward function encourages proximity of the system-level velocity + to a desired velocity specified in the environment parameters, while + slightly penalizing small time headways among autonomous vehicles. + Termination + A rollout is terminated if the time horizon is reached or if two + vehicles collide into one another. + """ + + def __init__(self, env_params, sim_params, network, simulator='traci'): + super().__init__(env_params, sim_params, network, simulator) + self.lead_obs = env_params.additional_params.get("lead_obs") + self.max_lanes = MAX_LANES + self.total_reward = 0.0 + + @property + def observation_space(self): + """See class definition.""" + # speed, speed of leader, headway + if self.lead_obs: + return Box( + low=-float('inf'), + high=float('inf'), + shape=(3 * MAX_NUM_VEHS,), + dtype=np.float32 + ) + # speed, dist to ego vehicle, binary value which is 1 if the vehicle is + # an AV + else: + leading_obs = 3 * self.max_lanes + follow_obs = 3 * self.max_lanes + + # speed and lane + self_obs = 2 + + return Box( + low=-float('inf'), + high=float('inf'), + shape=(leading_obs + follow_obs + self_obs,), + dtype=np.float32 + ) + + @property + def action_space(self): + """See class definition.""" + return Box( + low=-np.abs(self.env_params.additional_params['max_decel']), + high=self.env_params.additional_params['max_accel'], + shape=(1 * MAX_NUM_VEHS,), # (4,), + dtype=np.float32) + + def _apply_rl_actions(self, rl_actions): + """See class definition.""" + # in the warmup steps, rl_actions is None + if rl_actions is not None: + accels = [] + veh_ids = [] + rl_ids = self.get_sorted_rl_ids() + + for i, rl_id in enumerate(self.rl_id_list): + accels.append(rl_actions[i]) + veh_ids.append(rl_id) + + # lane_change_softmax = np.exp(actions[1:4]) + # lane_change_softmax /= np.sum(lane_change_softmax) + # lane_change_action = np.random.choice([-1, 0, 1], + # p=lane_change_softmax) + + self.k.vehicle.apply_acceleration(rl_ids, accels) + + def get_state(self): + """See class definition.""" + rl_ids = self.get_sorted_rl_ids() + self.rl_id_list = rl_ids + veh_info = np.zeros(self.observation_space.shape[0]) + per_vehicle_obs = 3 + for i, rl_id in enumerate(rl_ids): + speed = self.k.vehicle.get_speed(rl_id) + lead_id = self.k.vehicle.get_leader(rl_id) + if lead_id in ["", None]: + # in case leader is not visible + lead_speed = SPEED_SCALE + headway = HEADWAY_SCALE + else: + lead_speed = self.k.vehicle.get_speed(lead_id) + headway = self.k.vehicle.get_headway(rl_id) + veh_info[i * per_vehicle_obs: (i + 1) * per_vehicle_obs] = [speed / SPEED_SCALE, + headway / HEADWAY_SCALE, + lead_speed / SPEED_SCALE] + return veh_info + + def compute_reward(self, rl_actions, **kwargs): + """See class definition.""" + # in the warmup steps + if rl_actions is None: + return {} + + rl_ids = self.get_sorted_rl_ids() + + des_speed = self.env_params.additional_params["target_velocity"] + rewards = np.nan_to_num(np.mean([(des_speed - np.abs(speed - des_speed)) ** 2 + for speed in self.k.vehicle.get_speed(rl_ids)])) / (des_speed ** 2) + return rewards + + def get_sorted_rl_ids(self): + """Return the MAX_NUM_VEHS closest to the exit.""" + rl_ids = self.k.vehicle.get_rl_ids() + rl_ids = sorted(rl_ids, key=lambda veh_id: self.k.vehicle.get_x_by_id(veh_id)) + rl_ids = rl_ids[-MAX_NUM_VEHS:] + return rl_ids + + def additional_command(self): + """Define which vehicles are observed for visualization purposes.""" + # specify observed vehicles + for rl_id in self.k.vehicle.get_rl_ids(): + # leader + lead_id = self.k.vehicle.get_leader(rl_id) + if lead_id: + self.k.vehicle.set_observed(lead_id) + + def state_util(self, rl_id): + """Return an array of headway, tailway, leader speed, follower speed. + + Also return a 1 if leader is rl 0 otherwise, a 1 if follower is rl 0 otherwise. + If there are fewer than MAX_LANES the extra + entries are filled with -1 to disambiguate from zeros. + """ + veh = self.k.vehicle + lane_headways = veh.get_lane_headways(rl_id).copy() + lane_tailways = veh.get_lane_tailways(rl_id).copy() + lane_leader_speed = veh.get_lane_leaders_speed(rl_id).copy() + lane_follower_speed = veh.get_lane_followers_speed(rl_id).copy() + leader_ids = veh.get_lane_leaders(rl_id).copy() + follower_ids = veh.get_lane_followers(rl_id).copy() + rl_ids = self.k.vehicle.get_rl_ids() + is_leader_rl = [1 if l_id in rl_ids else 0 for l_id in leader_ids] + is_follow_rl = [1 if f_id in rl_ids else 0 for f_id in follower_ids] + diff = MAX_LANES - len(is_leader_rl) + if diff > 0: + # the minus 1 disambiguates missing cars from missing lanes + lane_headways += diff * [-1] + lane_tailways += diff * [-1] + lane_leader_speed += diff * [-1] + lane_follower_speed += diff * [-1] + is_leader_rl += diff * [-1] + is_follow_rl += diff * [-1] + lane_headways = np.asarray(lane_headways) / 1000 + lane_tailways = np.asarray(lane_tailways) / 1000 + lane_leader_speed = np.asarray(lane_leader_speed) / 100 + lane_follower_speed = np.asarray(lane_follower_speed) / 100 + return np.concatenate((lane_headways, lane_tailways, lane_leader_speed, + lane_follower_speed, is_leader_rl, + is_follow_rl)) + + def veh_statistics(self, rl_id): + """Return speed, edge information, and x, y about the vehicle itself.""" + speed = self.k.vehicle.get_speed(rl_id) / 100.0 + lane = (self.k.vehicle.get_lane(rl_id) + 1) / 10.0 + return np.array([speed, lane]) + + +class SingleStraightRoad(I210SingleEnv): + """Partially observable multi-agent environment for a straight road. Look at superclass for more information.""" + + def __init__(self, env_params, sim_params, network, simulator): + super().__init__(env_params, sim_params, network, simulator) + self.max_lanes = 1 + + def step(self, rl_actions): + """See parent class.""" + obs, rew, done, info = super().step(rl_actions) + mean_speed = np.nan_to_num(np.mean(self.k.vehicle.get_speed(self.k.vehicle.get_ids()))) + if self.env_params.additional_params['terminate_on_wave'] and \ + mean_speed < self.env_params.additional_params['wave_termination_speed'] \ + and self.time_counter > self.env_params.additional_params['wave_termination_horizon'] \ + and len(self.k.vehicle.get_ids()) > 0: + done = True + + return obs, rew, done, info + + +class SingleStraightRoadEnergyBracketing(I210SingleEnv): + """Partially observable multi-agent environment for a straight road. + + Refer to superclass for more information. Used here to run the simulation and + collect total energy consumption as a reward (no learning is applied). + """ + + # Initialize a highway envirnmoment with one lane + def __init__(self, env_params, sim_params, network, simulator): + super().__init__(env_params, sim_params, network, simulator) + self.max_lanes = 1 + + # Define actions taken every step + def step(self, rl_actions): + """See parent class.""" + obs, rew, done, info = super().step(rl_actions) + # check if all vehicles have exited the envirnoment + done = True + if self.step_counter < self.env_params.horizon and len(self.k.vehicle.get_ids()) > 0: + done = False + + return obs, rew, done, info + + def compute_reward(self, rl_actions, **kwargs): + """See class definition. + + Reward is defined as the total fuel consumption of the vehicle in the network at each time step + """ + gallons_per_step = 0 + # sum fuel consumption of each vehicle + for veh_id in self.k.vehicle.get_ids(): + energy_model = self.k.vehicle.get_energy_model(veh_id) + speed = self.k.vehicle.get_speed(veh_id) + accel = self.k.vehicle.get_accel(veh_id, noise=False, failsafe=True) + grade = self.k.vehicle.get_road_grade(veh_id) + gallons_per_hr = energy_model.get_instantaneous_fuel_consumption(accel, speed, grade) + gallons_per_step += gallons_per_hr * self.sim_step/3600 + return gallons_per_step + + def _apply_rl_actions(self, rl_actions): + """See class definition.""" + # in the warmup steps or if there is no rl veh, rl_actions is None + if rl_actions is not None: + accels = [] + veh_ids = [] + rl_ids = self.get_sorted_rl_ids() + # apply acceleration comaand on the controlled vehicle + for i, rl_id in enumerate(self.rl_id_list): + accels.append(rl_actions[i]) + veh_ids.append(rl_id) + + self.k.vehicle.apply_acceleration(rl_ids, accels) diff --git a/flow/networks/highway.py b/flow/networks/highway.py index e48331cf9..a9a282662 100644 --- a/flow/networks/highway.py +++ b/flow/networks/highway.py @@ -154,6 +154,12 @@ def specify_routes(self, net_params): """See parent class.""" num_edges = net_params.additional_params.get("num_edges", 1) rts = {} + + # Add the ghost edge (if needed). + if self.net_params.additional_params["use_ghost_edge"]: + rts.update({"highway_end": ["highway_end"]}) + + # Add the remaining edges. for i in range(num_edges): rts["highway_{}".format(i)] = ["highway_{}".format(j) for j in range(i, num_edges)] diff --git a/flow/networks/i210_subnetwork.py b/flow/networks/i210_subnetwork.py index f4315b07f..100e2247a 100644 --- a/flow/networks/i210_subnetwork.py +++ b/flow/networks/i210_subnetwork.py @@ -104,7 +104,42 @@ def specify_routes(self, net_params): "119257908#1-AddedOffRampEdge", "119257908#2", "119257908#3"], 1.0), - ] + ], + + # Main highway + "119257908#0": [ + (["119257908#0", + "119257908#1-AddedOnRampEdge", + "119257908#1", + "119257908#1-AddedOffRampEdge", + "119257908#2", + "119257908#3"], 1.0), + ], + "119257908#1-AddedOnRampEdge": [ + (["119257908#1-AddedOnRampEdge", + "119257908#1", + "119257908#1-AddedOffRampEdge", + "119257908#2", + "119257908#3"], 1.0), + ], + "119257908#1": [ + (["119257908#1", + "119257908#1-AddedOffRampEdge", + "119257908#2", + "119257908#3"], 1.0), + ], + "119257908#1-AddedOffRampEdge": [ + (["119257908#1-AddedOffRampEdge", + "119257908#2", + "119257908#3"], 1.0), + ], + "119257908#2": [ + (["119257908#2", + "119257908#3"], 1), + ], + "119257908#3": [ + (["119257908#3"], 1), + ], } if net_params.additional_params["ghost_edge"]: @@ -123,41 +158,6 @@ def specify_routes(self, net_params): if net_params.additional_params["on_ramp"]: rts.update({ - # Main highway - "119257908#0": [ - (["119257908#0", - "119257908#1-AddedOnRampEdge", - "119257908#1", - "119257908#1-AddedOffRampEdge", - "119257908#2", - "119257908#3"], 1.0), - ], - "119257908#1-AddedOnRampEdge": [ - (["119257908#1-AddedOnRampEdge", - "119257908#1", - "119257908#1-AddedOffRampEdge", - "119257908#2", - "119257908#3"], 1.0), - ], - "119257908#1": [ - (["119257908#1", - "119257908#1-AddedOffRampEdge", - "119257908#2", - "119257908#3"], 1.0), - ], - "119257908#1-AddedOffRampEdge": [ - (["119257908#1-AddedOffRampEdge", - "119257908#2", - "119257908#3"], 1.0), - ], - "119257908#2": [ - (["119257908#2", - "119257908#3"], 1), - ], - "119257908#3": [ - (["119257908#3"], 1), - ], - # On-ramp "27414345": [ (["27414345", diff --git a/flow/replay/hbaselines_replay.py b/flow/replay/hbaselines_replay.py new file mode 100644 index 000000000..a8f7dfc67 --- /dev/null +++ b/flow/replay/hbaselines_replay.py @@ -0,0 +1,354 @@ +"""Visualizer for rllib experiments. + +Attributes +---------- +EXAMPLE_USAGE : str + Example call to the function, which is + :: + + python ./visualizer_rllib.py /tmp/ray/result_dir 1 +""" +import tensorflow as tf +import numpy as np +import argparse +import os +import sys +import random +import json + +from flow.core.experiment import Experiment + +from hbaselines.fcnet.td3 import FeedForwardPolicy \ + as TD3FeedForwardPolicy +from hbaselines.fcnet.sac import FeedForwardPolicy \ + as SACFeedForwardPolicy +from hbaselines.goal_conditioned.td3 import GoalConditionedPolicy \ + as TD3GoalConditionedPolicy +from hbaselines.goal_conditioned.sac import GoalConditionedPolicy \ + as SACGoalConditionedPolicy +from hbaselines.envs.mixed_autonomy.params.highway \ + import get_flow_params as highway +from hbaselines.envs.mixed_autonomy.params.i210 \ + import get_flow_params as i210 + +EXAMPLE_USAGE = """ +example usage: + python ./visualizer_rllib.py /ray_results/experiment_dir/result_dir 1 + +Here the arguments are: +1 - the path to the simulation results +2 - the number of the checkpoint +""" + +# dictionary that maps policy names to policy objects +POLICY_DICT = { + "FeedForwardPolicy": { + "TD3": TD3FeedForwardPolicy, + "SAC": SACFeedForwardPolicy, + }, + "GoalConditionedPolicy": { + "TD3": TD3GoalConditionedPolicy, + "SAC": SACGoalConditionedPolicy, + }, +} + + +ENV_ATTRIBUTES = { + "highway-v0": lambda multiagent: highway( + fixed_boundary=True, + stopping_penalty=True, + acceleration_penalty=True, + multiagent=multiagent, + ), + "highway-v1": lambda multiagent: highway( + fixed_boundary=True, + stopping_penalty=False, + acceleration_penalty=True, + multiagent=multiagent, + ), + "highway-v2": lambda multiagent: highway( + fixed_boundary=True, + stopping_penalty=False, + acceleration_penalty=False, + multiagent=multiagent, + ), + "i210-v0": lambda multiagent: i210( + fixed_boundary=True, + stopping_penalty=True, + acceleration_penalty=True, + multiagent=multiagent, + ), + "i210-v1": lambda multiagent: i210( + fixed_boundary=True, + stopping_penalty=False, + acceleration_penalty=True, + multiagent=multiagent, + ), + "i210-v2": lambda multiagent: i210( + fixed_boundary=True, + stopping_penalty=False, + acceleration_penalty=False, + multiagent=multiagent, + ), +} + + +def parse_options(args): + """Parse training options user can specify in command line. + + Returns + ------- + argparse.Namespace + the output parser object + """ + parser = argparse.ArgumentParser( + description='Run evaluation episodes of a given checkpoint.', + epilog='python run_eval "/path/to/dir_name" ckpt_num') + + # required input parameters + parser.add_argument( + 'dir_name', type=str, help='the path to the checkpoints folder') + + # optional arguments + parser.add_argument( + '--ckpt_num', type=int, default=None, + help='the checkpoint number. If not specified, the last checkpoint is ' + 'used.') + parser.add_argument( + '--num_rollouts', type=int, default=1, + help='number of eval episodes') + parser.add_argument( + '--no_render', action='store_true', + help='shuts off rendering') + parser.add_argument( + '--random_seed', action='store_true', + help='whether to run the simulation on a random seed. If not added, ' + 'the original seed is used.') + parser.add_argument( + '--to_aws', action='store_true', + help='shuts off rendering') + + flags, _ = parser.parse_known_args(args) + + return flags + + +def get_hyperparameters_from_dir(ckpt_path): + """Collect the algorithm-specific hyperparameters from the checkpoint. + + Parameters + ---------- + ckpt_path : str + the path to the checkpoints folder + + Returns + ------- + str + environment name + hbaselines.goal_conditioned.* + policy object + dict + algorithm and policy hyperparaemters + int + the seed value + """ + # import the dictionary of hyperparameters + with open(os.path.join(ckpt_path, 'hyperparameters.json'), 'r') as f: + hp = json.load(f) + + # collect the policy object + policy_name = hp['policy_name'] + alg_name = hp['algorithm'] + policy = POLICY_DICT[policy_name][alg_name] + + # collect the environment name + env_name = hp['env_name'] + + # collect the seed value + seed = hp['seed'] + + # remove unnecessary features from hp dict + hp = hp.copy() + del hp['policy_name'], hp['env_name'], hp['seed'] + del hp['algorithm'], hp['date/time'] + + return env_name, policy, hp, seed + + +def get_flow_params(env_name): + """Read the provided result_dir and get config and flow_params. + + Parameters + ---------- + env_name : str + the name of the environment (in h-baselines) + + Returns + ------- + dict + the flow-params dict object + """ + # Handle multi-agent environments. + multiagent = env_name.startswith("multiagent") + if multiagent: + env_name = env_name[11:] + + flow_params = ENV_ATTRIBUTES[env_name](multiagent) + + dir_path = os.path.dirname(os.path.realpath(__file__)) + emission_path = '{0}/test_time_rollout/'.format(dir_path) + flow_params["sim"].emission_path = emission_path + + flow_params["sim"].render = True + + return flow_params + + +def get_rl_actions(policy_tf, ob_space): + """Define the method through which actions are assigned to the RL agent(s). + + Parameters + ---------- + policy_tf : hbaselines.base_policies.* + the policy object + ob_space : gym.space.* + the observation space + + Returns + ------- + method + the rl_actions method to use in the Experiment object + """ + def rl_actions(obs): + """Get the actions from a given observation. + + Parameters + ---------- + obs : array_like + the observation + + Returns + ------- + list of float + the action value + """ + # Reshape the observation to match the input structure of the policy. + if isinstance(obs, dict): + # In multi-agent environments, observations come in dict form + for key in obs.keys(): + # Shared policies with have one observation space, while + # independent policies have a different observation space based + # on their agent ID. + if isinstance(ob_space, dict): + ob_shape = ob_space[key].shape + else: + ob_shape = ob_space.shape + obs[key] = np.array(obs[key]).reshape((-1,) + ob_shape) + else: + obs = np.array(obs).reshape((-1,) + ob_space.shape) + + action = policy_tf.get_action( + obs, None, + apply_noise=False, + random_actions=False, + env_num=0, + ) + + # Flatten the actions. Dictionaries correspond to multi-agent policies. + if isinstance(action, dict): + action = {key: action[key].flatten() for key in action.keys()} + else: + action = action.flatten() + + return action + + return rl_actions + + +def main(args): + """Visualizer for RLlib experiments. + + This function takes args (see function create_parser below for + more detailed information on what information can be fed to this + visualizer), and renders the experiment associated with it. + """ + # ======================================================================= # + # Step 1: Import relevant data. # + # ======================================================================= # + + flags = parse_options(args) + + # get the hyperparameters + env_name, policy, hp, seed = get_hyperparameters_from_dir(flags.dir_name) + hp['num_envs'] = 1 + + # Get the flow-specific parameters. + flow_params = get_flow_params(env_name) + + # setup the seed value + if not flags.random_seed: + random.seed(seed) + np.random.seed(seed) + tf.compat.v1.set_random_seed(seed) + + # Create the experiment object. + exp = Experiment(flow_params) + + # ======================================================================= # + # Step 2: Setup the policy. # + # ======================================================================= # + + # Create a tensorflow session. + sess = tf.compat.v1.Session() + + # Get the checkpoint number. + if flags.ckpt_num is None: + filenames = os.listdir(os.path.join(flags.dir_name, "checkpoints")) + metafiles = [f[:-5] for f in filenames if f[-5:] == ".meta"] + metanum = [int(f.split("-")[-1]) for f in metafiles] + ckpt_num = max(metanum) + else: + ckpt_num = flags.ckpt_num + + # location to the checkpoint + ckpt = os.path.join(flags.dir_name, "checkpoints/itr-{}".format(ckpt_num)) + + # Create the policy. + policy_tf = policy( + sess=sess, + ob_space=exp.env.observation_space, + ac_space=exp.env.action_space, + co_space=None, + verbose=2, + layers=[256, 256], + act_fun=tf.nn.relu, + fingerprint_range=([0, 0], [5, 5]), + env_name=env_name, + **hp["policy_kwargs"] + ) + + trainable_vars = tf.compat.v1.get_collection( + tf.compat.v1.GraphKeys.TRAINABLE_VARIABLES) + + # Restore the previous checkpoint. + saver = tf.compat.v1.train.Saver(trainable_vars) + saver.restore(sess, ckpt) + + # Create a method to compute the RL actions. + rl_actions = get_rl_actions(policy_tf, exp.env.observation_space) + + # ======================================================================= # + # Step 3: Setup the and run the experiment. # + # ======================================================================= # + + exp.run( + num_runs=flags.num_rollouts, + convert_to_csv=True, + to_aws=flags.to_aws, + rl_actions=rl_actions, + multiagent=env_name.startswith("multiagent"), + ) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/flow/replay/rllib_replay.py b/flow/replay/rllib_replay.py new file mode 100644 index 000000000..f86715a16 --- /dev/null +++ b/flow/replay/rllib_replay.py @@ -0,0 +1,453 @@ +"""Replay script for rllib experiments. + +Attributes +---------- +EXAMPLE_USAGE : str + Example call to the function, which is + :: + + python ./rl_replay.py /tmp/ray/result_dir 1 +""" +import argparse +import gym +import numpy as np +from collections import defaultdict +import os +import sys + +import ray +try: + from ray.rllib.agents.agent import get_agent_class +except ImportError: + from ray.rllib.agents.registry import get_agent_class +from ray.tune.registry import register_env + +from flow.utils.rllib import get_flow_params +from flow.utils.rllib import get_rllib_config +from flow.utils.rllib import get_rllib_pkl + +from flow.core.experiment import Experiment + +EXAMPLE_USAGE = """ +example usage: + python ./rl_replay.py /ray_results/experiment_dir/result_dir 1 + +Here the arguments are: +1 - the path to the simulation results +2 - the number of the checkpoint +""" + + +def read_result_dir(result_dir_path): + """Read the provided result_dir and get config and flow_params. + + Parameters + ---------- + result_dir_path : str + Directory containing rllib results + + Returns + ------- + str + the path to the results directory + dict + TODO + bool + True if the agent is multi-agent, False otherwise + dict + flow-specific parameters from the results directory + """ + result_dir = result_dir_path if result_dir_path[-1] != '/' \ + else result_dir_path[:-1] + + config = get_rllib_config(result_dir) + + # check if we have a multiagent environment but in a + # backwards compatible way + if config.get('multiagent', {}).get('policies', None): + multiagent = True + pkl = get_rllib_pkl(result_dir) + config['multiagent'] = pkl['multiagent'] + else: + multiagent = False + + # Run on only one cpu for rendering purposes + config['num_workers'] = 0 + + flow_params = get_flow_params(config) + return result_dir, config, multiagent, flow_params + + +def set_sim_params(sim_params, render_mode, save_render, gen_emission, output_dir=''): + """Set up sim_params according to render mode. + + Parameters + ---------- + sim_params : flow.core.params.SimParams + simulation-specific parameters + render_mode : str + the render mode. Options include sumo_web3d, rgbd and sumo_gui + save_render : bool + Saves a rendered video to a file. NOTE: Overrides render_mode with + pyglet rendering. + gen_emission : bool + whether to generate the emission file + output_dir : str + directory to store the emission, optional + """ + # hack for old pkl files TODO(ev) remove eventually + setattr(sim_params, 'num_clients', 1) + if not hasattr(sim_params, 'use_ballistic'): + sim_params.use_ballistic = False + + # Set the emission path. + sim_params.restart_instance = True + if not output_dir: + dir_path = os.path.dirname(os.path.realpath(__file__)) + emission_path = '{0}/test_time_rollout/'.format(dir_path) + else: + emission_path = output_dir + sim_params.emission_path = emission_path if gen_emission else None + + # Pick your rendering mode. + if render_mode == 'sumo_web3d': + sim_params.num_clients = 2 + sim_params.render = False + elif render_mode == 'drgb': + sim_params.render = 'drgb' + sim_params.pxpm = 4 + elif render_mode == 'sumo_gui': + sim_params.render = False # will be set to True below + elif render_mode == 'no_render': + sim_params.render = False + + # Add option for saving rendered results. + if save_render: + if render_mode != 'sumo_gui': + sim_params.render = 'drgb' + sim_params.pxpm = 4 + sim_params.save_render = True + + +def set_env_params(env_params, evaluate, horizon, config=None): + """Set up env_params according to commandline arguments. + + Parameters + ---------- + env_params : flow.core.params.EnvParams + environment-specific parameters + evaluate : bool + flag indicating that the evaluation reward should be used so the + evaluation reward should be used rather than the normal reward + horizon : int + an update environment time horizon. If set to None, the original time + horizon is used. + config : dict + TODO + """ + # Start the environment with the gui turned on and a path for the + # emission file + env_params.restart_instance = False + if evaluate: + env_params.evaluate = True + + # lower the horizon if testing + if horizon: + if config: + config['horizon'] = horizon + env_params.horizon = horizon + + +def set_agents(config, result_dir, env_name, run=None, checkpoint_num=None): + """Determine and create agents that will be used to compute actions. + + Parameters + ---------- + config : dict + TODO + result_dir : str + Directory containing rllib results + env_name : str + the name of the environment + run : str + The algorithm or model to train. This may refer to the name of a + built-on algorithm (e.g. RLLib's DQN or PPO), or a user-defined + trainable function or class registered in the tune registry. Required + for results trained with flow-0.2.0 and before. + checkpoint_num : int + the checkpoint number. If set to None, the most recent checkpoint is + used. + + Returns + ------- + TODO + the trained agent/policy object + """ + # Determine agent and checkpoint + config_run = config['env_config']['run'] if 'run' in config['env_config'] \ + else None + if run and config_run: + if run != config_run: + print('rl_replay.py: error: run argument ' + + '\'{}\' passed in '.format(run) + + 'differs from the one stored in params.json ' + + '\'{}\''.format(config_run)) + sys.exit(1) + if run: + agent_cls = get_agent_class(run) + elif config['env_config']['run'] == \ + "": + from flow.algorithms.centralized_PPO import CCTrainer + from flow.algorithms.centralized_PPO import CentralizedCriticModel + from ray.rllib.models import ModelCatalog + agent_cls = CCTrainer + ModelCatalog.register_custom_model("cc_model", CentralizedCriticModel) + elif config['env_config']['run'] == \ + "": + from flow.algorithms.custom_ppo import CustomPPOTrainer + agent_cls = CustomPPOTrainer + elif config_run: + agent_cls = get_agent_class(config_run) + else: + print('rl_replay.py: error: could not find flow parameter ' + '\'run\' in params.json, ' + 'add argument --run to provide the algorithm or model used ' + 'to train the results\n e.g. ' + 'python ./rl_replay.py /tmp/ray/result_dir 1 --run PPO') + sys.exit(1) + + # get checkpoint number if not provided + if checkpoint_num is None: + dirs = os.listdir(result_dir) + cp_numbers = [int(name.split('_')[1]) for name in dirs if name.startswith('checkpoint')] + checkpoint_num = str(max(cp_numbers)) + + # create the agent that will be used to compute the actions + agent = agent_cls(env=env_name, config=config) + checkpoint = result_dir + '/checkpoint_' + checkpoint_num + checkpoint = checkpoint + '/checkpoint-' + checkpoint_num + agent.restore(checkpoint) + + return agent + + +def get_rl_action(config, agent, multiagent, multi_only=False, deterministic=False): + """Return a function that compute action based on a given state. + + Parameters + ---------- + config : dict + TODO + agent : TODO + the trained agent/policy object + multiagent : bool + whether the policy is a multi-agent policy + multi_only : bool + If this is set to true, the function will raise an error + if it is single agent use_lstm is true. + deterministic : bool + If this is set to true, the exploration noise in the policy + will be removed when computing actions. + + Returns + ------- + policy_map_fn : function + a mapping from agent to their respective policy + rl_action : method + the rl_actions method to use in the Experiment object + rets : dict + a pre-initialized dictionary to store rewards for multi-agent simulation + """ + policy_map_fn = None + if multiagent: + rets = {} + # map the agent id to its policy + policy_map_fn = config['multiagent']['policy_mapping_fn'] + for key in config['multiagent']['policies'].keys(): + rets[key] = [] + else: + rets = [] + + if config['model']['use_lstm']: + use_lstm = True + if multiagent: + state_init = {} + size = config['model']['lstm_cell_size'] + state_init = defaultdict(lambda: [np.zeros(size, np.float32), + np.zeros(size, np.float32)]) + else: + state_init = [ + np.zeros(config['model']['lstm_cell_size'], np.float32), + np.zeros(config['model']['lstm_cell_size'], np.float32) + ] + else: + use_lstm = False + + def rl_action(state): + if multiagent: + action = {} + for agent_id in state.keys(): + if use_lstm: + action[agent_id], state_init[agent_id], logits = \ + agent.compute_action( + state[agent_id], state=state_init[agent_id], + policy_id=policy_map_fn(agent_id)) + else: + action[agent_id] = agent.compute_action(state[agent_id], + policy_id=policy_map_fn(agent_id)) + else: + if use_lstm and multi_only: + raise NotImplementedError + action = agent.compute_action(state, explore=not deterministic) + return action + + return policy_map_fn, rl_action, rets + + +def replay_rllib(args): + """Replay for RLlib experiments. + + This function takes args (see function create_parser below for + more detailed information on what information can be fed to this + replay script), and renders the experiment associated with it. + """ + result_dir, config, multiagent, flow_params = read_result_dir( + args.result_dir) + + set_sim_params( + flow_params['sim'], + args.render_mode, + args.save_render, + args.gen_emission + ) + sim_params = flow_params['sim'] + + # Create and register a gym+rllib env + exp = Experiment(flow_params, register_with_ray=True) + register_env(exp.env_name, exp.create_env) + + set_env_params(flow_params['env'], args.evaluate, args.horizon, config) + + agent = set_agents( + config, + result_dir, + exp.env_name, + run=args.run, + checkpoint_num=args.checkpoint_num + ) + + if hasattr(agent, "local_evaluator") and \ + os.environ.get("TEST_FLAG") != 'True': + exp.env = agent.local_evaluator.env + else: + exp.env = gym.make(exp.env_name) + + # reroute on exit is a training hack, it should be turned off at test time. + if hasattr(exp.env, "reroute_on_exit"): + exp.env.reroute_on_exit = False + + # Set to True after initializing agent and env. + if args.render_mode == 'sumo_gui': + exp.env.sim_params.render = True + + policy_map_fn, rl_action, rets = get_rl_action(config, agent, multiagent, + deterministic=args.deterministic) + + # If restart_instance, don't restart here because env.reset will restart + # later. + if not sim_params.restart_instance: + exp.env.restart_simulation(sim_params, sim_params.render) + + exp.run( + num_runs=args.num_rollouts, + convert_to_csv=args.gen_emission, + to_aws=args.to_aws, + rl_actions=rl_action, + multiagent=multiagent, + rets=rets, + policy_map_fn=policy_map_fn + ) + + +def create_parser(): + """Create the parser to capture CLI arguments.""" + parser = argparse.ArgumentParser( + description='[Flow] Evaluates a reinforcement learning agent ' + 'given a checkpoint.', + epilog=EXAMPLE_USAGE) + + # required input parameters + parser.add_argument( + 'result_dir', type=str, help='Directory containing results') + parser.add_argument('checkpoint_num', type=str, help='Checkpoint number.') + + # optional input parameters + parser.add_argument( + '--run', + type=str, + help='The algorithm or model to train. This may refer to ' + 'the name of a built-on algorithm (e.g. RLLib\'s DQN ' + 'or PPO), or a user-defined trainable function or ' + 'class registered in the tune registry. ' + 'Required for results trained with flow-0.2.0 and before.') + parser.add_argument( + '--num_rollouts', + type=int, + default=1, + help='The number of rollouts to replay.') + parser.add_argument( + '--gen_emission', + action='store_true', + help='Specifies whether to generate an emission file from the ' + 'simulation') + parser.add_argument( + '--evaluate', + action='store_true', + help='Specifies whether to use the \'evaluate\' reward ' + 'for the environment.') + parser.add_argument( + '--render_mode', + type=str, + default='sumo_gui', + help='Pick the render mode. Options include sumo_web3d, ' + 'rgbd and sumo_gui') + parser.add_argument( + '--save_render', + action='store_true', + help='Saves a rendered video to a file. NOTE: Overrides render_mode ' + 'with pyglet rendering.') + parser.add_argument( + '--horizon', + type=int, + help='Specifies the horizon.') + parser.add_argument( + '--is_baseline', + action='store_true', + help='specifies whether this is a baseline run' + ) + parser.add_argument( + '--to_aws', + type=str, nargs='?', default=None, const="default", + help='Specifies the name of the partition to store the output' + 'file on S3. Putting not None value for this argument' + 'automatically set gen_emission to True.' + ) + parser.add_argument( + '--deterministic', + action='store_true', + default=False, + help='specifies whether to remove exploration noise in the policy' + ) + return parser + + +def main(): + """Run the rl_replay according to the commandline arguments.""" + parser = create_parser() + args = parser.parse_args() + ray.init(num_cpus=1) + replay_rllib(args) + + +if __name__ == '__main__': + main() diff --git a/flow/replay/transfer_tests.py b/flow/replay/transfer_tests.py new file mode 100644 index 000000000..7b958d1d3 --- /dev/null +++ b/flow/replay/transfer_tests.py @@ -0,0 +1,604 @@ +"""Transfer and replay for i210 environment.""" +import argparse +import numpy as np +import os +from copy import deepcopy + +from flow.controllers.car_following_models import IDMController, SimCarFollowingController +from flow.controllers.rlcontroller import RLController +from flow.controllers.velocity_controllers import FollowerStopper + +from examples.exp_configs.rl.multiagent.multiagent_i210 import flow_params as I210_MA_DEFAULT_FLOW_PARAMS +from examples.exp_configs.rl.multiagent.multiagent_i210 import custom_callables +from flow.core.experiment import Experiment +from flow.envs.multiagent.i210 import I210TransferEnv +from flow.utils.registry import make_create_env +from flow.visualize.transfer.util import inflows_range +from flow.replay.rllib_replay import read_result_dir +from flow.replay.rllib_replay import set_sim_params +from flow.replay.rllib_replay import set_env_params +from flow.replay.rllib_replay import set_agents +from flow.replay.rllib_replay import get_rl_action +import ray +from ray.tune.registry import register_env + +from flow.data_pipeline.data_pipeline import collect_metadata_from_config + +EXAMPLE_USAGE = """ +example usage: + python transfer_tests.py -r /ray_results/experiment_dir/result_dir -c 1 + python transfer_tests.py --controller idm + python transfer_tests.py --controller idm --run_transfer + +Here the arguments are: +1 - the path to the simulation results +2 - the number of the checkpoint +""" + + +@ray.remote +def replay(args, + flow_params, + output_dir=None, + transfer_test=None, + rllib_config=None, + max_completed_trips=None, + supplied_metadata=None, + inflow_rate=None, + outflow_speed_limit=None, + controller_params={}, + default_controller_params={}): + """Replay or run transfer test (defined by transfer_fn) by modif. + + Parameters + ---------- + args : argparse.Namespace + input arguments passed via a parser. See create_parser. + flow_params : dict + flow-specific parameters + output_dir : str + Directory to save results. + transfer_test : TODO + TODO + rllib_config : str + Directory containing rllib results + max_completed_trips : int + Terminate rollout after max_completed_trips vehicles have started and + ended. + supplied_metadata: dict (str : list) + the metadata associated with this simulation. + inflow_rate : int + The inflow rate. Analogous to what you see in multiagent_i210's exp_config + outflow_speed_limit : int + The speed limit of the final edge + controller_params : dict + Custom params for the once-RL vehicle. e.g. If you replace the RL vehicles with + IDM vehicles, this could be IDM params + default_controller_params : dict + Custom params for the once-human vehicle e.g. If you replace the human vehicles with + IDM vehicles, this could be IDM params + """ + args.gen_emission = args.gen_emission or args.use_s3 + + if transfer_test is not None: + if type(transfer_test) == bytes: + transfer_test = ray.cloudpickle.loads(transfer_test) + flow_params = transfer_test.flow_params_modifier_fn(flow_params) + + # Choose the controller to replace the AV, as well as initialize with any + # controller-specific parameters + if args.controller: + test_params = {} + if args.controller == 'idm': + controller = IDMController + if 'idm_params' in controller_params: + test_params.update(controller_params['idm_params']) + elif args.controller == 'default_human': + controller = flow_params['veh'].type_parameters['human']['acceleration_controller'][0] + test_params.update(flow_params['veh'].type_parameters['human']['acceleration_controller'][1]) + elif args.controller == 'follower_stopper': + controller = FollowerStopper + if 'v_des' in controller_params: + test_params.update({'v_des': controller_params['v_des']}) + else: + test_params.update({'v_des': 12}) + elif args.controller == 'sumo': + controller = SimCarFollowingController + # Update the parameters + flow_params['veh'].type_parameters['rl']['acceleration_controller'] = (controller, test_params) + for veh_param in flow_params['veh'].initial: + if veh_param['veh_id'] == 'rl': + veh_param['acceleration_controller'] = (controller, test_params) + + # Choose the controller to replace the original human controller, as well as initialize with any + # controller-specific parameters + if args.default_controller: + test_params = {} + if args.default_controller == 'idm': + default_controller = IDMController + # If the original human-controlled vehicle was IDM, first initialize with its + # parameters before applying other modifications. + if flow_params['veh'].type_parameters['human']['acceleration_controller'][0] == IDMController: + test_params.update(flow_params['veh'].type_parameters['human']['acceleration_controller'][1]) + if 'idm_params' in default_controller_params: + test_params.update(default_controller_params['idm_params']) + elif args.default_controller == 'follower_stopper': + default_controller = FollowerStopper + # If the original human-controlled vehicle was FollowerStopper, first initialize with its + # parameters before applying other modifications. + if flow_params['veh'].type_parameters['human']['acceleration_controller'][0] == FollowerStopper: + test_params.update(flow_params['veh'].type_parameters['human']['acceleration_controller'][1]) + if 'v_des' in default_controller_params: + test_params.update({'v_des': default_controller_params['v_des']}) + else: + test_params.update({'v_des': 12}) + elif args.default_controller == 'sumo': + default_controller = SimCarFollowingController + + # Update the parameters + flow_params['veh'].type_parameters['human']['acceleration_controller'] = (default_controller, test_params) + for veh_param in flow_params['veh'].initial: + if veh_param['veh_id'] == 'human': + veh_param['acceleration_controller'] = (default_controller, test_params) + + # Updates the values of the default IDM controller applied to once-human vehicles. + if args.idm_sweep: + test_params = {} + test_params.update(flow_params['veh'].type_parameters['human']['acceleration_controller'][1]) + test_params.update(default_controller_params['idm_params']) + default_controller = IDMController + + # Update the parameters + flow_params['veh'].type_parameters['human']['acceleration_controller'] = (default_controller, test_params) + for veh_param in flow_params['veh'].initial: + if veh_param['veh_id'] == 'human': + veh_param['acceleration_controller'] = (default_controller, test_params) + + # Modifies the lcSpeedGain parameter of once-human vehicles + if args.lane_freq_sweep: + test_lane_params = {'lcSpeedGain': str(float(default_controller_params['lane_frequency']))} + + flow_params['veh'].type_parameters['human']['lane_change_params'].controller_params.update(test_lane_params) + for veh_param in flow_params['veh'].initial: + if veh_param['veh_id'] == 'human': + veh_param['lane_change_params'].controller_params.update(test_lane_params) + for vtype in flow_params['veh'].types: + if vtype['veh_id'] == 'human': + vtype['type_params']['lcSpeedGain'] = str(float(default_controller_params['lane_frequency'])) + + # TODO is there a more dynamic way instead of hardcoding the start edges + # Modifies the inflow rate + if args.inflow_sweep: + from examples.exp_configs.rl.multiagent.multiagent_i210 import PENETRATION_RATE, \ + ON_RAMP, WANT_BOUNDARY_CONDITIONS, ON_RAMP_INFLOW_RATE, ENTER_AS_LINE + for inflow in flow_params['net'].inflows.get(): + if ENTER_AS_LINE: + if WANT_BOUNDARY_CONDITIONS: + if inflow['edge'] == 'ghost0': + if inflow['vtype'] == 'human': + inflow['vehsPerHour'] = int(inflow_rate * (1 - PENETRATION_RATE)) + elif inflow['vtype'] == 'rl': + inflow['vehsPerHour'] = int(inflow_rate * PENETRATION_RATE) + else: + if inflow['edge'] == "119257914": + if inflow['vtype'] == 'human': + inflow['vehsPerHour'] = int(inflow_rate * (1 - PENETRATION_RATE)) + elif inflow['vtype'] == 'rl': + inflow['vehsPerHour'] = int(inflow_rate * PENETRATION_RATE) + if ON_RAMP: + if inflow['edge'] == '27414345' or inflow['edge'] == "27414342#0": + inflow['vehsPerHour'] = int(ON_RAMP_INFLOW_RATE * (1 - PENETRATION_RATE)) + + else: + if WANT_BOUNDARY_CONDITIONS: + if inflow['edge'] == 'ghost0': + if inflow['vtype'] == 'human': + inflow['vehsPerHour'] = int(inflow_rate * 5 * (1 - PENETRATION_RATE)) + elif inflow['vtype'] == 'rl': + inflow['vehsPerHour'] = int(inflow_rate * 5 * PENETRATION_RATE) + else: + if inflow['edge'] == '119257914': + if inflow['vtype'] == 'human': + inflow['vehsPerHour'] = int(inflow_rate * 5 * (1 - PENETRATION_RATE)) + elif inflow['vtype'] == 'rl': + inflow['vehsPerHour'] = int(inflow_rate * 5 * PENETRATION_RATE) + + if ON_RAMP: + if inflow['edge'] == '27414345' or inflow['edge'] == '27414342#0': + inflow['vehsPerHour'] = int(ON_RAMP_INFLOW_RATE * (1 - PENETRATION_RATE)) + + # Modifies the speed limit of the last edge + if args.outflow_sweep: + flow_params['env'].additional_params['outflow_speed_limit'] = outflow_speed_limit + + set_sim_params(flow_params['sim'], args.render_mode, args.save_render, args.gen_emission, output_dir) + sim_params = flow_params['sim'] + + set_env_params(flow_params['env'], args.evaluate, args.horizon) + + # Create and register a gym+rllib env + exp = Experiment(flow_params, custom_callables=custom_callables) + + # set to True after initializing agent and env + if args.render_mode == 'sumo_gui': + exp.env.sim_params.render = True + + # if restart_instance, don't restart here because env.reset will restart later + if not sim_params.restart_instance: + exp.env.restart_simulation(sim_params=sim_params, render=sim_params.render) + + # reroute on exit is a training hack, it should be turned off at test time. + if hasattr(exp.env, "reroute_on_exit"): + exp.env.reroute_on_exit = False + + policy_map_fn, rets = None, None + if rllib_config: + result_dir, rllib_config, multiagent, rllib_flow_params = read_result_dir(rllib_config) + + # lower the horizon if testing + if args.horizon: + rllib_config['horizon'] = args.horizon + + agent_create_env, agent_env_name = make_create_env(params=rllib_flow_params, version=0) + register_env(agent_env_name, agent_create_env) + + assert 'run' in rllib_config['env_config'], \ + "Was this trained with the latest version of Flow?" + # Determine agent and checkpoint + agent = set_agents(rllib_config, result_dir, agent_env_name, checkpoint_num=args.checkpoint_num) + + policy_map_fn, rllib_rl_action, rets = get_rl_action(rllib_config, agent, multiagent) + + # reroute on exit is a training hack, it should be turned off at test time. + if hasattr(exp.env, "reroute_on_exit"): + exp.env.reroute_on_exit = False + + def rl_action(state): + if rllib_config: + action = rllib_rl_action(state) + else: + action = None + return action + + info_dict = exp.run( + num_runs=args.num_rollouts, + convert_to_csv=args.gen_emission, + to_aws=args.use_s3, + rl_actions=rl_action, + multiagent=True, + rets=rets, + policy_map_fn=policy_map_fn, + supplied_metadata=supplied_metadata + ) + + return info_dict + + +def create_parser(): + """Create the parser to capture CLI arguments.""" + parser = argparse.ArgumentParser( + description='[Flow] Evaluates a reinforcement learning agent ' + 'given a checkpoint.', + epilog=EXAMPLE_USAGE) + + parser.add_argument( + '--rllib_result_dir', '-r', + required=False, + type=str, + help='Directory containing results' + ) + parser.add_argument( + '--checkpoint_num', '-c', + required=False, + type=str, + default=None, + help='Checkpoint number.' + ) + parser.add_argument( + '--num_rollouts', + type=int, + default=1, + help='The number of rollouts to visualize.') + parser.add_argument( + '--gen_emission', + action='store_true', + help='Specifies whether to generate an emission file from the ' + 'simulation') + parser.add_argument( + '--evaluate', + action='store_true', + help='Specifies whether to use the \'evaluate\' reward ' + 'for the environment.') + parser.add_argument( + '--render_mode', '-rm', + type=str, + default=None, + help='Pick the render mode. Options include sumo_web3d, ' + 'rgbd and sumo_gui') + parser.add_argument( + '--save_render', + action='store_true', + help='Saves a rendered video to a file. NOTE: Overrides render_mode ' + 'with pyglet rendering.') + parser.add_argument( + '--horizon', + type=int, + help='Specifies the horizon.') + parser.add_argument( + '--local', + action='store_true', + help='Adjusts run settings to be compatible with limited ' + 'memory capacity' + ) + parser.add_argument( + '--controller', + type=str, + help='Which custom controller to use. Defaults to IDM' + ) + parser.add_argument( + '--default_controller', + type=str, + help='Which controller the non-controlled vehicles use. Defaults to IDM' + ) + parser.add_argument( + '--run_transfer', + action='store_true', + help='Runs transfer tests if true' + ) + parser.add_argument( + '-pr', + '--penetration_rate', + type=float, + help='Specifies percentage of AVs.', + required=False) + parser.add_argument( + '-mct', + '--max_completed_trips', + type=int, + help='Terminate rollout after max_completed_trips vehicles have ' + 'started and ended.', + default=None) + parser.add_argument( + '--v_des_sweep', + action='store_true', + help='Runs a sweep over v_des params.', + default=None) + parser.add_argument( + '--idm_sweep', + action='store_true', + help='Runs a sweep over idm params.', + default=None) + parser.add_argument( + '--inflow_sweep', + action='store_true', + help='Runs a sweep over the inflows of other vehicles.', + default=None) + parser.add_argument( + '--outflow_sweep', + action='store_true', + help='Runs a sweep over the outflow speed.', + default=None) + parser.add_argument( + '--lane_freq_sweep', + action='store_true', + help='Runs a sweep over the lane change frequencies.', + default=None) + parser.add_argument( + '--output_dir', + type=str, + help='Directory to save results.', + default=None + ) + parser.add_argument( + '--use_s3', + action='store_true', + help='If true, upload results to s3' + ) + parser.add_argument( + '--num_cpus', + type=int, + default=1, + help='Number of cpus to run experiment with' + ) + parser.add_argument( + '--multi_node', + action='store_true', + help='Set to true if this will be run in cluster mode' + ) + parser.add_argument( + '--exp_title', + type=str, + required=False, + default=None, + help='Informative experiment title to help distinguish results' + ) + parser.add_argument( + '--only_query', + nargs='*', default="[\'all\']", + help='specify which query should be run by lambda for detail, see ' + 'upload_to_s3 in data_pipeline.py' + ) + parser.add_argument( + '--is_baseline', + action='store_true', + help='specifies whether this is a baseline run' + ) + parser.add_argument( + '--no_warmup', + action='store_true', + help='Set warmup steps to 0. Mostly for debugging purposes' + ) + parser.add_argument( + '--no_warnings', + action='store_true', + help='Sets display_warnings to False in the vehicle definition' + ) + parser.add_argument( + '--exp_config', type=str, + help='Name of the experiment configuration file, as located in ' + 'exp_configs/non_rl.') + parser.add_argument( + '--submitter_name', type=str, + help='Name displayed next to the submission on the leaderboard.') + parser.add_argument( + '--strategy_name', type=str, + help='Strategy displayed next to the submission on the leaderboard.') + return parser + + +def generate_graphs(args): + """Run the replay according to the commandline arguments.""" + supplied_metadata = None + if args.submitter_name and args.strategy_name: + supplied_metadata = {'name': args.submitter_name, + 'strategy': args.strategy_name} + if args.exp_config: + module = __import__("../../examples/exp_configs.non_rl", fromlist=[args.exp_config]) + flow_params = getattr(module, args.exp_config).flow_params + supplied_metadata = collect_metadata_from_config(getattr(module, args.exp_config)) + else: + flow_params = deepcopy(I210_MA_DEFAULT_FLOW_PARAMS) + # Use the TestEnv + flow_params['env_name'] = I210TransferEnv + if args.no_warmup: + flow_params['env'].warmup_steps = 0 + + if args.no_warnings: + for type_param in flow_params['veh'].type_parameters.values(): + if not type_param['acceleration_controller'][0] == RLController: + type_param['acceleration_controller'][1]['display_warnings'] = False + for veh_param in flow_params['veh'].initial: + if not type_param['acceleration_controller'][0] == RLController: + veh_param['acceleration_controller'][1]['display_warnings'] = False + + if ray.is_initialized(): + ray.shutdown() + if args.multi_node: + ray.init(redis_address='localhost:6379') + elif args.local: + ray.init(local_mode=True, object_store_memory=200 * 1024 * 1024) + else: + ray.init(num_cpus=args.num_cpus, object_store_memory=200 * 1024 * 1024) + + if args.exp_title: + output_dir = os.path.join(args.output_dir, args.exp_title) + else: + output_dir = args.output_dir + + if args.run_transfer: + s = [ray.cloudpickle.dumps(transfer_test) for transfer_test in + inflows_range(penetration_rates=[0.0, 0.1, 0.2, 0.3])] + ray_output = [ + replay.remote( + args, + flow_params, + output_dir=output_dir, + transfer_test=transfer_test, + rllib_config=args.rllib_result_dir, + max_completed_trips=args.max_completed_trips, + supplied_metadata=supplied_metadata + ) + for transfer_test in s + ] + ray.get(ray_output) + + elif args.v_des_sweep: + assert args.controller == 'follower_stopper' + + ray_output = [ + replay.remote( + args, + flow_params, + output_dir="{}/{}".format(output_dir, v_des), + rllib_config=args.rllib_result_dir, + max_completed_trips=args.max_completed_trips, + supplied_metadata=supplied_metadata, + controller_params={'v_des': v_des} + ) + for v_des in range(8, 13, 1) + ] + ray.get(ray_output) + + elif args.idm_sweep: + assert args.default_controller == 'idm' + ray_output = [ + replay.remote( + args, + flow_params, + output_dir="{}/{}".format(output_dir, a), + rllib_config=args.rllib_result_dir, + max_completed_trips=args.max_completed_trips, + default_controller_params={'idm_params': {'a': a}} + ) + for a in np.arange(0.2, 2, 0.2)] + ray.get(ray_output) + + elif args.inflow_sweep: + ray_output = [ + replay.remote( + args, + flow_params, + output_dir="{}/{}".format(output_dir, inflow_rate), + rllib_config=args.rllib_result_dir, + max_completed_trips=args.max_completed_trips, + inflow_rate=inflow_rate + ) + for inflow_rate in range(1800, 2500, 100)] + ray.get(ray_output) + + elif args.outflow_sweep: + ray_output = [ + replay.remote( + args, + flow_params, + output_dir="{}/{}".format(output_dir, outflow_speed_limit), + rllib_config=args.rllib_result_dir, + max_completed_trips=args.max_completed_trips, + outflow_speed_limit=outflow_speed_limit + ) + for outflow_speed_limit in range(2, 10, 2)] + ray.get(ray_output) + + elif args.lane_freq_sweep: + assert args.default_controller == 'idm' + ray_output = [ + replay.remote( + args, + flow_params, + output_dir="{}/{}".format(output_dir, lf), + rllib_config=args.rllib_result_dir, + max_completed_trips=args.max_completed_trips, + default_controller_params={'lane_frequency': lf} + ) + for lf in [1, 10, 20]] + ray.get(ray_output) + + else: + if args.penetration_rate is not None: + pr = args.penetration_rate + single_transfer = next(inflows_range(penetration_rates=pr)) + ray.get(replay.remote( + args, + flow_params, + output_dir=output_dir, + transfer_test=single_transfer, + rllib_config=args.rllib_result_dir, + max_completed_trips=args.max_completed_trips, + supplied_metadata=supplied_metadata + )) + else: + ray.get(replay.remote( + args, + flow_params, + output_dir=output_dir, + rllib_config=args.rllib_result_dir, + max_completed_trips=args.max_completed_trips, + supplied_metadata=supplied_metadata + )) + + +if __name__ == "__main__": + parser = create_parser() + args = parser.parse_args() + + generate_graphs(args) diff --git a/flow/utils/registry.py b/flow/utils/registry.py index 3f6c9dad5..a1163849f 100644 --- a/flow/utils/registry.py +++ b/flow/utils/registry.py @@ -117,15 +117,19 @@ def create_env(*_): entry_point = params["env_name"].__module__ + ':' + params["env_name"].__name__ # register the environment with OpenAI gym - register( - id=env_name, - entry_point=entry_point, - kwargs={ - "env_params": env_params, - "sim_params": sim_params, - "network": network, - "simulator": params['simulator'] - }) + try: + register( + id=env_name, + entry_point=entry_point, + kwargs={ + "env_params": env_params, + "sim_params": sim_params, + "network": network, + "simulator": params['simulator'] + }) + except gym.error.Error: + print("WARNING: Environment {} already registered, ignoring." + .format(env_name)) return gym.envs.make(env_name) diff --git a/flow/utils/rllib.py b/flow/utils/rllib.py index 80193c22b..db0e811b8 100644 --- a/flow/utils/rllib.py +++ b/flow/utils/rllib.py @@ -8,6 +8,7 @@ import os import sys +import flow.config import flow.envs from flow.core.params import SumoLaneChangeParams, SumoCarFollowingParams, \ SumoParams, InitialConfig, EnvParams, NetParams, InFlows @@ -96,6 +97,8 @@ def get_flow_params(config): flow_params = json.loads(config['env_config']['flow_params']) else: flow_params = json.load(open(config, 'r')) + if 'env_config' in flow_params: + flow_params = json.loads(flow_params['env_config']['flow_params']) # reinitialize the vehicles class from stored data veh = VehicleParams() @@ -146,6 +149,12 @@ def get_flow_params(config): if flow_params["net"]["inflows"]: net.inflows.__dict__ = flow_params["net"]["inflows"].copy() + if net.template is not None and len(net.template) > 0: + filename = os.path.join(flow.config.PROJECT_PATH, 'examples') + split = net.template.split('examples')[1][1:] + path = os.path.abspath(os.path.join(filename, split)) + net.template = path + env = EnvParams() env.__dict__ = flow_params["env"].copy() diff --git a/flow/visualize/energy_model_verification.csv b/flow/visualize/energy_model_verification.csv new file mode 100644 index 000000000..5b46a6c30 --- /dev/null +++ b/flow/visualize/energy_model_verification.csv @@ -0,0 +1,78 @@ +road_grade(%),acceleration(m/s^2),speed(m/s),Tacoma(g/s),Tacoma_fit(g/s),Tacoma(gal/hr),Tacoma_fit(gal/hr),Tacoma(KW),Tacoma_fit(KW),Prius(SOC %/s),Prius_fit(KW) +0,-3,0,0.221939015830998,0.202898145738508,0.248342670601163,0.22703654508,3.72514005901745,3.4055481762,0,0.00104601 +0,-3,5,0.221939015830998,0,0.248342670601163,0,3.72514005901745,0,-3.54871940694454e-05,-11.69 +0,-3,10,0,0,0,0,0,0,-7.69535592950981e-05,-23.38 +0,-3,15,0,0,0,0,0,0,-0.000116458349774967,-35.07 +0,-3,20,0,0,0,0,0,0,-0.000153287003370266,-46.76 +0,-3,25,0,0,0,0,0,0,-0.000188081089779002,-58.45 +0,-3,30,0,0,0,0,0,0,-0.000220255167062945,-70.14 +0,-2.5,0,0.221939015830998,0.202898145738508,0.248342670601163,0.22703654508,3.72514005901745,3.4055481762,0,0.00104601 +0,-2.5,5,0.221939015830998,0,0.248342670601163,0,3.72514005901745,0,-3.54871940694454e-05,-11.69 +0,-2.5,10,0,0,0,0,0,0,-7.69535592950981e-05,-23.38 +0,-2.5,15,0,0,0,0,0,0,-0.000116458349774967,-35.07 +0,-2.5,20,0,0,0,0,0,0,-0.000153287003370266,-46.76 +0,-2.5,25,0,0,0,0,0,0,-0.000188081089779002,-58.45 +0,-2.5,30,0,0,0,0,0,0,-0.000220255167062945,-70.14 +0,-2,0,0.221939015830998,0.202898145738508,0.248342670601163,0.22703654508,3.72514005901745,3.4055481762,0,0.00104601 +0,-2,5,0.221939015830998,0,0.248342670601163,0,3.72514005901745,0,-3.54871940694454e-05,-11.69 +0,-2,10,0,0,0,0,0,0,-7.69535592950981e-05,-23.38 +0,-2,15,0,0,0,0,0,0,-0.000116458349774967,-35.07 +0,-2,20,0,0,0,0,0,0,-0.000153287003370266,-46.76 +0,-2,25,0,0,0,0,0,0,-0.000188081089779002,-58.45 +0,-2,30,0,0,0,0,0,0,-0.000219898832587135,-70.14 +0,-1.5,0,0.221939015830998,0.202898145738508,0.248342670601163,0.22703654508,3.72514005901745,3.4055481762,0,0.00104601 +0,-1.5,5,0.221939015830998,0,0.248342670601163,0,3.72514005901745,0,-3.25561591348804e-05,-10.271045160915 +0,-1.5,10,0,0,0,0,0,0,-6.63944522434643e-05,-20.27904442157 +0,-1.5,15,0,0,0,0,0,0,-9.75752995002611e-05,-29.773774871775 +0,-1.5,20,0,0,0,0,0,0,-0.000125409125499782,-38.50592258403 +0,-1.5,25,0,0,0,0,0,0,-0.000149176326217658,-46.226173630835 +0,-1.5,30,0,0,0,0,0,0,-0.000168098899055845,-52.68521408469 +0,-1,0,0.221939015830998,0.202898145738508,0.248342670601163,0.22703654508,3.72514005901745,3.4055481762,0,0.00104601 +0,-1,5,0.221939015830998,0,0.248342670601163,0,3.72514005901745,0,-2.20548454687974e-05,-6.658177660915 +0,-1,10,0,0,0,0,0,0,-4.41187234444278e-05,-13.05330942157 +0,-1,15,0,0,0,0,0,0,-6.40570164847835e-05,-18.935172371775 +0,-1,20,0,0,0,0,0,0,-8.11312166655779e-05,-24.05445258403 +0,-1,25,0,0,0,0,0,0,-9.46028296724449e-05,-28.161836130835 +0,-1,30,0,0,0,0,0,0,-0.000103742492341937,-31.00800908469 +0,-0.5,0,0.221939015830998,0.202898145738508,0.248342670601163,0.22703654508,3.72514005901745,3.4055481762,0,0.00104601 +0,-0.5,5,0.221939015830998,0,0.248342670601163,0,3.72514005901745,0,-1.05375387210715e-05,-3.045310160915 +0,-0.5,10,0,0,0,0,0,0,-2.03458170463669e-05,-5.82757442157 +0,-0.5,15,0,0,0,0,0,0,-2.83134794828346e-05,-8.096569871775 +0,-0.5,20,0,0,0,0,0,0,-3.36349054121418e-05,-9.60298258403 +0,-0.5,25,0,0,0,0,0,0,-3.54943418155165e-05,-10.097498630835 +0,-0.5,30,0,0.0229074063027033,0,0.0256326560540001,0,0.384489840810002,-3.30813003554099e-05,-9.33080408469 +0,0,0,0.200487388049937,0.202898145738508,0.224338984219372,0.22703654508,3.36508476329058,3.4055481762,0,0.00104601 +0,0,5,0.25334167260449,0.242980516160659,0.283481240617348,0.271887437463167,4.25221860926022,4.0783115619475,2.03157739236523e-06,0.653115465 +0,0,10,0.333366925874865,0.334679208358044,0.373026943243302,0.374495345431333,5.59540414864953,5.61743018147,5.01788707628445e-06,1.60893047 +0,0,15,0.512564850241743,0.509457833595245,0.573543697527477,0.5700670451295,8.60315546291215,8.5510056769425,9.85521374467623e-06,3.155388525 +0,0,20,0.799694695426426,0.798780003136848,0.894832824161981,0.893809312702667,13.4224923624297,13.40713969054,1.74571556614643e-05,5.57938713 +0,0,25,1.22925671358626,1.23410932824744,1.37549900346897,1.38092892429583,20.6324850520345,20.7139338644375,2.8807542951233e-05,9.167823785 +0,0,30,1.85724065673253,1.84690942019159,2.0781929797924,2.066632656054,31.172894696886,30.99948984081,4.49825609596261e-05,14.20759599 +0,0.5,0,0.213329615431394,0.339890899834573,0.238709026514264,0.380327061746667,3.58063539771397,5.7049059262,0,0.00104601 +0,0.5,5,0.645372234433188,0.82921564072096,0.722150918939926,0.927865819129833,10.8322637840989,13.9179872869475,1.5671887972443e-05,5.554765465 +0,0.5,10,0.962084652592057,1.37015670338258,1.0765419999476,1.533161592098,16.148129999214,22.99742388147,3.20484965256772e-05,11.41223047 +0,0.5,15,1.49224306419028,1.99417769908402,1.66977233074365,2.23142115679617,25.0465849611548,33.4713173519425,5.06379365955613e-05,17.860338525 +0,0.5,20,2.17179531854029,2.73274223908986,2.43016960035591,3.05785128936933,36.4525440053386,45.86776934054,7.25377933928332e-05,25.18598713 +0,0.5,25,2.99467055233709,3.61731393466468,3.35094070663254,4.0476587659625,50.2641105994881,60.7148814894375,9.90452111270652e-05,33.676073785 +0,0.5,30,4.33709133305878,4.67935639707307,4.85306668708108,5.23605036272067,72.7960003062162,78.54075544081,0.00013178402890794,43.61749599 +0,1,0,0.352945791629779,0.476883653930638,0.394935068728636,0.533617578413333,5.92402603092953,8.0042636762,0,0.00104601 +0,1,5,1.06813790168415,1.41545076528126,1.1952121986364,1.5838442007965,17.928182979546,23.7576630119475,3.04163065457829e-05,10.456415465 +0,1,10,1.71955641506992,2.40563419840712,1.92412871062296,2.69182783876467,28.8619306593444,40.37741758147,6.08497873003605e-05,21.21553047 +0,1,15,2.57963132066376,3.47889756457279,2.88652506158662,3.89277526846283,43.2978759237992,58.3916290269425,9.4286160430679e-05,32.565288525 +0,1,20,3.59296715103925,4.66670447504287,4.02041549265406,5.221893266036,60.3062323898109,78.32839899054,0.00013213988251624,44.79258713 +0,1,25,4.96053483428683,6.00051854108193,5.55068005390709,6.71438860762917,83.2602008086064,100.715829114437,0.000176265302590978,58.184323785 +0,1,30,8.4686721949014,7.51180337395456,9.47617372030307,8.40546806938733,142.142605804546,126.08202104081,0.000229310382602672,73.02739599 +0,1.5,0,0.546883865191755,0.613876408026703,0.611945579202834,0.68690809508,9.17918368804252,10.3036214262,0,0.00104601 +0,1.5,5,1.59178526738712,2.00168588984156,1.78115687702032,2.23982258246317,26.7173531553048,33.5973387369475,4.63101243713366e-05,15.358065465 +0,1.5,10,2.47488397540235,3.44111169343166,2.76931612757742,3.85049408543133,41.5397419136613,57.75741128147,9.15563557882123e-05,31.01883047 +0,1.5,15,3.76403627779172,4.96361743006157,4.21183638201887,5.5541293801295,63.177545730283,83.3119407019425,0.000141129468194144,47.270238525 +0,1.5,20,5.51088858527308,6.60066671099588,6.16650832449549,7.38593524270267,92.4976248674323,110.78902864054,0.000196988846466955,64.39918713 +0,1.5,25,7.87103521022029,8.38372314749917,8.80743702130485,9.38111844929583,132.111555319573,140.716776739437,0.000261977398069592,82.692573785 +0,1.5,30,11.007333064259,10.344250350836,12.3168541553602,11.574885776054,184.752812330403,173.62328664081,0.000340664672572568,102.43729599 +0,2,0,0.771261491371735,0.750869162122768,0.863016976902089,0.840198611746667,12.9452546535313,12.6029791762,0,0.00104601 +0,2,5,2.18406638946791,2.58792101440186,2.4439005368201,2.89580096412983,36.6585080523015,43.4370144619475,6.33953886952215e-05,20.259715465 +0,2,10,3.68364029601661,4.47658918845619,4.12187584603615,5.009160332098,61.8281376905423,75.13740498147,0.000124309559813017,40.82213047 +0,2,15,6.18127824468203,6.44833729555034,6.91665294299644,7.21548349179617,103.749794144947,108.232252376943,0.000191549200096262,61.975188525 +0,2,20,10.6807256375028,8.53462894694889,11.9513908757511,9.54997721936933,179.270863136266,143.24965829054,0.000268006912217915,84.00578713 +0,2,25,18.8504578823379,10.7669277539164,21.0930603392389,12.0478482909625,316.395905088583,180.717724364438,0.000358314214701827,107.200823785 +0,2,30,13.4802447146304,13.1766973277175,15.0839633142185,14.7443034827207,226.259449713277,221.16455224081,0.000470854478779554,131.84719599 diff --git a/flow/visualize/plot_custom_callables.py b/flow/visualize/plot_custom_callables.py new file mode 100644 index 000000000..ee9a10c1d --- /dev/null +++ b/flow/visualize/plot_custom_callables.py @@ -0,0 +1,115 @@ +"""Generate charts from with .npy files containing custom callables through replay.""" + +import argparse +from datetime import datetime +import errno +import numpy as np + +try: + from matplotlib import pyplot as plt +except ImportError: + import matplotlib + + matplotlib.use('TkAgg') + from matplotlib import pyplot as plt +import os +import pytz +import sys + + +def make_bar_plot(vals, title): + """Make a bar plot.""" + print(len(vals)) + fig = plt.figure() + plt.hist(vals, 10, facecolor='blue', alpha=0.5) + plt.title(title) + plt.xlim(1000, 3000) + return fig + + +def plot_trip_distribution(all_trip_energy_distribution): + """Plot a distribution of trips.""" + non_av_vals = [] + figures = [] + figure_names = [] + for key in all_trip_energy_distribution: + if key != 'av': + non_av_vals.extend(all_trip_energy_distribution[key]) + figures.append(make_bar_plot(all_trip_energy_distribution[key], key)) + figure_names.append(key) + + figure_names.append('All Non-AV') + figures.append(make_bar_plot(non_av_vals, 'All Non-AV')) + + figure_names.append('All') + figures.append(make_bar_plot(non_av_vals + all_trip_energy_distribution['av'], 'All')) + + return figure_names, figures + + +def parse_flags(args): + """Parse training options user can specify in command line. + + Returns + ------- + argparse.Namespace + the output parser object + """ + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description="Parse argument used when running a Flow simulation.", + epilog="python train.py EXP_CONFIG") + parser.add_argument("target_folder", type=str, + help='Folder containing results') + parser.add_argument("--output_folder", type=str, required=False, default=None, + help='Folder to save charts to.') + parser.add_argument("--show_images", action='store_true', + help='Whether to display charts.') + return parser.parse_args(args) + + +if __name__ == "__main__": + flags = parse_flags(sys.argv[1:]) + + date = datetime.now(tz=pytz.utc) + date = date.astimezone(pytz.timezone('US/Pacific')).strftime("%m-%d-%Y") + + if flags.output_folder: + if not os.path.exists(flags.output_folder): + try: + os.makedirs(flags.output_folder) + except OSError as exc: + if exc.errno != errno.EEXIST: + raise + + info_dicts = [] + custom_callable_names = set() + exp_names = [] + for (dirpath, dir_names, file_names) in os.walk(flags.target_folder): + for file_name in file_names: + if file_name[-4:] == ".npy": + exp_name = os.path.basename(dirpath) + info_dict = np.load(os.path.join(dirpath, file_name), allow_pickle=True).item() + + info_dicts.append(info_dict) + exp_names.append(exp_name) + custom_callable_names.update(info_dict.keys()) + + idxs = np.argsort(exp_names) + exp_names = [exp_names[i] for i in idxs] + info_dicts = [info_dicts[i] for i in idxs] + + for name in custom_callable_names: + y_vals = [np.mean(info_dict[name]) for info_dict in info_dicts] + y_stds = [np.std(info_dict[name]) for info_dict in info_dicts] + x_pos = np.arange(len(exp_names)) + + plt.bar(x_pos, y_vals, align='center', alpha=0.5) + plt.xticks(x_pos, [exp_name for exp_name in exp_names], rotation=60) + plt.xlabel('Experiment') + plt.title('I210 Replay Result: {}'.format(name)) + plt.tight_layout() + if flags.output_folder: + plt.savefig(os.path.join(flags.output_folder, '{}-plot.png'.format(name))) + + plt.show() diff --git a/flow/visualize/time_space_diagram.py b/flow/visualize/time_space_diagram.py index 4914fc6a7..9ae005712 100644 --- a/flow/visualize/time_space_diagram.py +++ b/flow/visualize/time_space_diagram.py @@ -25,7 +25,7 @@ from matplotlib import pyplot as plt except ImportError: import matplotlib - matplotlib.use('TkAgg') + matplotlib.use('Agg') from matplotlib import pyplot as plt from matplotlib.collections import LineCollection, PatchCollection from matplotlib.patches import Rectangle @@ -43,6 +43,17 @@ HighwayNetwork ] +# networks that use edgestarts +USE_EDGESTARTS = set([ + RingNetwork, + FigureEightNetwork, + MergeNetwork +]) + +GHOST_DICT = defaultdict(dict) +GHOST_DICT[I210SubNetwork] = {'ghost_edges': {'ghost0', '119257908#3'}} +GHOST_DICT[HighwayNetwork] = {'ghost_bounds': (500, 2300)} + def import_data_from_trajectory(fp, params=dict()): r"""Import and preprocess data from the Flow trajectory (.csv) file. @@ -53,7 +64,6 @@ def import_data_from_trajectory(fp, params=dict()): file path (for the .csv formatted file) params : dict flow-specific parameters, including: - * "network" (str): name of the network that was used when generating the emission file. Must be one of the network names mentioned in ACCEPTABLE_NETWORKS, @@ -63,8 +73,10 @@ def import_data_from_trajectory(fp, params=dict()): Returns ------- - pd.DataFrame + pd.DataFrame, float, float """ + network = params['network'] + # Read trajectory csv into pandas dataframe df = pd.read_csv(fp) @@ -74,34 +86,47 @@ def import_data_from_trajectory(fp, params=dict()): 'lane_number': 'lane_id', } df = df.rename(columns=column_conversions) - if 'distance' not in df.columns: + if network in USE_EDGESTARTS: df['distance'] = _get_abs_pos(df, params) + start = params['env'].warmup_steps * params['env'].sims_per_step * params['sim'].sim_step + # produce upper and lower bounds for the non-greyed-out domain + ghost_edges = GHOST_DICT[network].get('ghost_edges') + ghost_bounds = GHOST_DICT[network].get('ghost_bounds') + if ghost_edges: + domain_lb = df[~df['edge_id'].isin(ghost_edges)]['distance'].min() + domain_ub = df[~df['edge_id'].isin(ghost_edges)]['distance'].max() + elif ghost_bounds: + domain_lb = ghost_bounds[0] + domain_ub = ghost_bounds[1] + else: + domain_lb = df['distance'].min() + domain_ub = df['distance'].max() + + df.loc[:, 'time_step'] = df['time_step'].apply(lambda x: x - start) + df.loc[:, 'distance'] = df['distance'].apply(lambda x: x - domain_lb) + domain_ub -= domain_lb + # Compute line segment ends by shifting dataframe by 1 row df[['next_pos', 'next_time']] = df.groupby('id')[['distance', 'time_step']].shift(-1) # Remove nans from data df = df[df['next_time'].notna()] - return df + return df, domain_lb, domain_ub, start -def get_time_space_data(data, params): +def get_time_space_data(data, network): r"""Compute the unique inflows and subsequent outflow statistics. Parameters ---------- data : pd.DataFrame cleaned dataframe of the trajectory data - params : dict - flow-specific parameters, including: - - * "network" (str): name of the network that was used when generating - the emission file. Must be one of the network names mentioned in - ACCEPTABLE_NETWORKS, - * "net_params" (flow.core.params.NetParams): network-specific - parameters. This is used to collect the lengths of various network - links. + network : child class of Network() + network that was used when generating the emission file. + Must be one of the network names mentioned in + ACCEPTABLE_NETWORKS Returns ------- @@ -109,7 +134,6 @@ def get_time_space_data(data, params): 3d array (n_segments x 2 x 2) containing segments to be plotted. every inner 2d array is comprised of two 1d arrays representing [start time, start distance] and [end time, end distance] pairs. - in the case of I210, the nested arrays are wrapped into a dict, keyed on the lane number, so that each lane can be plotted separately. @@ -120,8 +144,8 @@ def get_time_space_data(data, params): if the specified network is not supported by this method """ # check that the network is appropriate - assert params['network'] in ACCEPTABLE_NETWORKS, \ - 'Network must be one of: ' + ', '.join([network.__name__ for network in ACCEPTABLE_NETWORKS]) + assert network in ACCEPTABLE_NETWORKS, \ + 'Network must be one of: ' + ', '.join([network_.__name__ for network_ in ACCEPTABLE_NETWORKS]) # switcher used to compute the positions based on the type of network switcher = { @@ -133,7 +157,7 @@ def get_time_space_data(data, params): } # Get the function from switcher dictionary - func = switcher[params['network']] + func = switcher[network] # Execute the function segs, data = func(data) @@ -212,6 +236,7 @@ def _ring_road(data): pd.DataFrame unmodified trajectory dataframe """ + data = data[data['next_pos'] > data['distance']] segs = data[['time_step', 'distance', 'next_time', 'next_pos']].values.reshape((len(data), 2, 2)) return segs, data @@ -241,7 +266,8 @@ def _i210_subnetwork(data): """ # Reset lane numbers that are offset by ramp lanes offset_edges = set(data[data['lane_id'] == 5]['edge_id'].unique()) - data.loc[data['edge_id'].isin(offset_edges), 'lane_id'] -= 1 + data.loc[~data['edge_id'].isin(offset_edges), 'lane_id'] = \ + data[~data['edge_id'].isin(offset_edges)]['lane_id'] + 1 segs = dict() for lane, df in data.groupby('lane_id'): @@ -385,79 +411,140 @@ def _get_abs_pos(df, params): return ret -def plot_tsd(ax, df, segs, args, lane=None, ghost_edges=None, ghost_bounds=None): +def plot_tsd(df, network, cmap, min_speed=0, max_speed=10, start=0, domain_bounds=None): """Plot the time-space diagram. - Take the pre-processed segments and other meta-data, then plot all the line segments. + Take the pre-processed segments and other meta-data, then plot all the line + segments. Parameters ---------- - ax : matplotlib.axes.Axes - figure axes that will be plotted on df : pd.DataFrame data used for axes bounds and speed coloring - segs : list of list of lists - line segments to be plotted, where each segment is a list of two [x,y] pairs - args : dict - parsed arguments - lane : int, optional - lane number to be shown in plot title - ghost_edges : list or set of str - ghost edge names to be greyed out, default None - ghost_bounds : tuple + network : child class of Network() + network that was used when generating the emission file. + Must be one of the network names mentioned in + ACCEPTABLE_NETWORKS + cmap : colors.LinearSegmentedColormap + colormap for plotting speed + min_speed : int or float + minimum speed in colorbar + max_speed : int or float + maximum speed in colorbar + start : int or float + starting time_step not greyed out + domain_bounds : tuple lower and upper bounds of domain, excluding ghost edges, default None - - Returns - ------- - None """ - norm = plt.Normalize(args.min_speed, args.max_speed) + norm = plt.Normalize(min_speed, max_speed) xmin, xmax = df['time_step'].min(), df['time_step'].max() xbuffer = (xmax - xmin) * 0.025 # 2.5% of range ymin, ymax = df['distance'].min(), df['distance'].max() ybuffer = (ymax - ymin) * 0.025 # 2.5% of range - ax.set_xlim(xmin - xbuffer, xmax + xbuffer) - ax.set_ylim(ymin - ybuffer, ymax + ybuffer) - - lc = LineCollection(segs, cmap=my_cmap, norm=norm) - lc.set_array(df['speed'].values) - lc.set_linewidth(1) - ax.add_collection(lc) - ax.autoscale() - - rects = [] - if ghost_edges: - y_domain_min = df[~df['edge_id'].isin(ghost_edges)]['distance'].min() - y_domain_max = df[~df['edge_id'].isin(ghost_edges)]['distance'].max() - rects.append(Rectangle((xmin, y_domain_min), args.start - xmin, y_domain_max - y_domain_min)) - rects.append(Rectangle((xmin, ymin), xmax - xmin, y_domain_min - ymin)) - rects.append(Rectangle((xmin, y_domain_max), xmax - xmin, ymax - y_domain_max)) - elif ghost_bounds: - rects.append(Rectangle((xmin, ghost_bounds[0]), args.start - xmin, ghost_bounds[1] - ghost_bounds[0])) - rects.append(Rectangle((xmin, ymin), xmax - xmin, ghost_bounds[0] - ymin)) - rects.append(Rectangle((xmin, ghost_bounds[1]), xmax - xmin, ymax - ghost_bounds[1])) - else: - rects.append(Rectangle((xmin, ymin), args.start - xmin, ymax - ymin)) + # Convert df data into segments for plotting + segs, df = get_time_space_data(df, network) + + nlanes = df['lane_id'].nunique() + plt.figure(figsize=(16, 9*nlanes)) + if nlanes == 1: + segs = [segs] + + for lane, lane_df in df.groupby('lane_id'): + ax = plt.subplot(nlanes, 1, lane+1) + + lc = LineCollection(segs[lane], cmap=cmap, norm=norm) + lc.set_array(lane_df['speed'].values) + lc.set_linewidth(1) + ax.add_collection(lc) + ax.autoscale() + + rects = [] + # rectangle for warmup period, but not ghost edges + rects.append(Rectangle((xmin, 0), start, domain_bounds[1])) + # rectangle for lower ghost edge (including warmup period) + rects.append(Rectangle((xmin, ymin), xmax - xmin, domain_bounds[0])) + # rectangle for upper ghost edge (including warmup period) + rects.append(Rectangle((xmin, domain_bounds[1]), xmax - xmin, ymax - domain_bounds[1])) - if rects: pc = PatchCollection(rects, facecolor='grey', alpha=0.5, edgecolor=None) pc.set_zorder(20) ax.add_collection(pc) - if lane: - ax.set_title('Time-Space Diagram: Lane {}'.format(lane), fontsize=25) - else: - ax.set_title('Time-Space Diagram', fontsize=25) - ax.set_ylabel('Position (m)', fontsize=20) - ax.set_xlabel('Time (s)', fontsize=20) - plt.xticks(fontsize=18) - plt.yticks(fontsize=18) + if nlanes > 1: + ax.set_title('Time-Space Diagram: Lane {}'.format(lane), fontsize=25) + else: + ax.set_title('Time-Space Diagram', fontsize=25) + + ax.set_xlim(xmin - xbuffer, xmax + xbuffer) + ax.set_ylim(ymin - ybuffer, ymax + ybuffer) + + ax.set_ylabel('Position (m)', fontsize=20) + if lane == nlanes - 1: + ax.set_xlabel('Time (s)', fontsize=20) + plt.xticks(fontsize=18) + plt.yticks(fontsize=18) + + cbar = plt.colorbar(lc, ax=ax, norm=norm) + cbar.set_label('Velocity (m/s)', fontsize=20) + cbar.ax.tick_params(labelsize=18) + + plt.tight_layout() + + +def tsd_main(trajectory_path, flow_params, min_speed=0, max_speed=10): + """Prepare and plot the time-space diagram. + + Parameters + ---------- + trajectory_path : str + file path (for the .csv formatted file) + flow_params : dict + flow-specific parameters, including: + * "network" (str): name of the network that was used when generating + the emission file. Must be one of the network names mentioned in + ACCEPTABLE_NETWORKS, + * "net_params" (flow.core.params.NetParams): network-specific + parameters. This is used to collect the lengths of various network + links. + min_speed : int or float + minimum speed in colorbar + max_speed : int or float + maximum speed in colorbar + """ + network = flow_params['network'] - cbar = plt.colorbar(lc, ax=ax, norm=norm) - cbar.set_label('Velocity (m/s)', fontsize=20) - cbar.ax.tick_params(labelsize=18) + # some plotting parameters + cdict = { + 'red': ((0, 0, 0), (0.2, 1, 1), (0.6, 1, 1), (1, 0, 0)), + 'green': ((0, 0, 0), (0.2, 0, 0), (0.6, 1, 1), (1, 1, 1)), + 'blue': ((0, 0, 0), (0.2, 0, 0), (0.6, 0, 0), (1, 0, 0)) + } + my_cmap = colors.LinearSegmentedColormap('my_colormap', cdict, 1024) + + # Read trajectory csv into pandas dataframe + traj_df, domain_lb, domain_ub, start = import_data_from_trajectory(trajectory_path, flow_params) + + plot_tsd(df=traj_df, + network=network, + cmap=my_cmap, + min_speed=min_speed, + max_speed=max_speed, + start=start, + domain_bounds=(domain_lb, domain_ub)) + + ########################################################################### + # Note: For MergeNetwork only # + if network == MergeNetwork: # + plt.plot([traj_df['time_step'].min(), traj_df['time_step'].max()], + [0, 0], linewidth=3, color="white") # + plt.plot([traj_df['time_step'].min(), traj_df['time_step'].max()], + [-0.1, -0.1], linewidth=3, color="white") # + ########################################################################### + + outfile = trajectory_path.replace('csv', 'png') + plt.savefig(outfile) if __name__ == '__main__': @@ -478,14 +565,11 @@ def plot_tsd(ax, df, segs, args, lane=None, ghost_edges=None, ghost_bounds=None) parser.add_argument('--steps', type=int, default=1, help='rate at which steps are plotted.') parser.add_argument('--title', type=str, default='Time Space Diagram', - help='rate at which steps are plotted.') + help='Title for the time-space diagrams.') parser.add_argument('--max_speed', type=int, default=8, help='The maximum speed in the color range.') parser.add_argument('--min_speed', type=int, default=0, help='The minimum speed in the color range.') - parser.add_argument('--start', type=float, default=0, - help='initial time (in sec) in the plot.') - args = parser.parse_args() # flow_params is imported as a dictionary @@ -495,47 +579,9 @@ def plot_tsd(ax, df, segs, args, lane=None, ghost_edges=None, ghost_bounds=None) module = __import__("examples.exp_configs.non_rl", fromlist=[args.flow_params]) flow_params = getattr(module, args.flow_params).flow_params - # some plotting parameters - cdict = { - 'red': ((0, 0, 0), (0.2, 1, 1), (0.6, 1, 1), (1, 0, 0)), - 'green': ((0, 0, 0), (0.2, 0, 0), (0.6, 1, 1), (1, 1, 1)), - 'blue': ((0, 0, 0), (0.2, 0, 0), (0.6, 0, 0), (1, 0, 0)) - } - my_cmap = colors.LinearSegmentedColormap('my_colormap', cdict, 1024) - - # Read trajectory csv into pandas dataframe - traj_df = import_data_from_trajectory(args.trajectory_path, flow_params) - - # Convert df data into segments for plotting - segs, traj_df = get_time_space_data(traj_df, flow_params) - - if flow_params['network'] == I210SubNetwork: - nlanes = traj_df['lane_id'].nunique() - fig = plt.figure(figsize=(16, 9*nlanes)) - - for lane, df in traj_df.groupby('lane_id'): - ax = plt.subplot(nlanes, 1, lane+1) - - plot_tsd(ax, df, segs[lane], args, int(lane+1), ghost_edges={'ghost0', '119257908#3'}) - plt.tight_layout() - else: - # perform plotting operation - fig = plt.figure(figsize=(16, 9)) - ax = plt.axes() - - if flow_params['network'] == HighwayNetwork: - plot_tsd(ax, traj_df, segs, args, ghost_bounds=(500, 2300)) - else: - plot_tsd(ax, traj_df, segs, args) - - ########################################################################### - # Note: For MergeNetwork only # - if flow_params['network'] == 'MergeNetwork': # - plt.plot([df['time_step'].min(), df['time_step'].max()], - [0, 0], linewidth=3, color="white") # - plt.plot([df['time_step'].min(), df['time_step'].max()], - [-0.1, -0.1], linewidth=3, color="white") # - ########################################################################### - - outfile = args.trajectory_path.replace('csv', 'png') - plt.savefig(outfile) + tsd_main( + args.trajectory_path, + flow_params, + min_speed=args.min_speed, + max_speed=args.max_speed + ) diff --git a/flow/visualize/transfer/util.py b/flow/visualize/transfer/util.py new file mode 100644 index 000000000..8c933c5a3 --- /dev/null +++ b/flow/visualize/transfer/util.py @@ -0,0 +1,141 @@ +"""Definitions of transfer classes.""" +from copy import deepcopy + +from flow.core.params import InFlows +from examples.exp_configs.rl.multiagent.multiagent_i210 import INFLOW_RATE, ON_RAMP_INFLOW_RATE + + +def make_inflows(pr=0.1, fr_coef=1.0, departSpeed=20, on_ramp=False): + """Generate inflows object from parameters. Uses default inflows from multiagent_i210. + + Keyword Arguments: + ----------------- + pr {float} -- [AV Penetration Rate] (default: {0.1}) + fr_coef {float} -- [Scale flow rate by] (default: {1.0}) + departSpeed {int} -- [Initial speed of all flows] (default: {20}) + + Returns + ------- + [Inflows] -- [Inflows parameter object] + + """ + inflow = InFlows() + # main highway + assert pr < 1.0, "your penetration rate is over 100%" + + all_inflows = [] + + inflow_119257914 = dict(veh_type="human", + edge="ghost0", + vehs_per_hour=INFLOW_RATE * (1 - (pr)) * fr_coef, + # probability=1.0, + departLane="random", + departSpeed=departSpeed) + all_inflows.append(inflow_119257914) + + if pr > 0.0: + inflow_119257914_av = dict(veh_type="av", + edge="ghost0", + vehs_per_hour=int(INFLOW_RATE * pr * fr_coef), + # probability=1.0, + departLane="random", + departSpeed=departSpeed) + all_inflows.append(inflow_119257914_av) + + if on_ramp: + inflow_27414345 = dict(veh_type="human", + edge="27414345", + vehs_per_hour=ON_RAMP_INFLOW_RATE * (1 - (pr)) * fr_coef, + departLane="random", + departSpeed=departSpeed) + all_inflows.append(inflow_27414345) + if pr > 0.0: + inflow_27414342 = dict(veh_type="human", + edge="27414342#0", + vehs_per_hour=ON_RAMP_INFLOW_RATE * pr * fr_coef, + departLane="random", + departSpeed=departSpeed) + all_inflows.append(inflow_27414342) + + for inflow_def in all_inflows: + inflow.add(**inflow_def) + + return inflow + + +class BaseTransfer: + """Base Transfer class.""" + + def __init__(self): + self.transfer_str = "Base" + pass + + def flow_params_modifier_fn(self, flow_params, clone_params=True): + """Return modified flow_params. + + Arguments: + --------- + flow_params {[flow_params_dictionary]} -- [flow_params] + """ + if clone_params: + flow_params = deepcopy(flow_params) + + return flow_params + + def env_modifier_fn(self, env): + """Modify the env before rollouts are run. + + Arguments: + --------- + env {[I210MultiEnv]} -- [Env to modify] + """ + pass + + +class InflowTransfer(BaseTransfer): + """Modifies the inflow of i210 env.""" + + def __init__(self, penetration_rate=0.1, flow_rate_coef=1.0, departSpeed=20): + super(InflowTransfer, self).__init__() + self.penetration_rate = penetration_rate + self.flow_rate_coef = flow_rate_coef + self.departSpeed = departSpeed + + self.transfer_str = "{:0.2f}_pen_{:0.2f}_flow_rate_coef_{:0.2f}_depspeed".format( + penetration_rate, flow_rate_coef, departSpeed) + + def flow_params_modifier_fn(self, flow_params, clone_params=True): + """See Parent.""" + if clone_params: + flow_params = deepcopy(flow_params) + + flow_params['net'].inflows = make_inflows(self.penetration_rate, self.flow_rate_coef, self.departSpeed) + + return flow_params + + +def inflows_range(penetration_rates=0.1, flow_rate_coefs=1.0, departSpeeds=20.0): + """Generate inflow objects given penetration_rates, flow_rates, and depart speeds. + + Keyword Arguments: + ----------------- + penetration_rates {float | list of floats} -- [single, or multiple penetration rates] (default: {0.1}) + flow_rate_coefs {float | list of floats} -- [single, or multiple flow rate coefficient] (default: {1.0}) + departSpeeds {float | list of floats} -- [single, or multiple depart speeds] (default: {20.0}) + + Yields + ------ + [InflowTransfer] -- [Transfer object] + """ + if not hasattr(penetration_rates, '__iter__'): + penetration_rates = [penetration_rates] + if not hasattr(flow_rate_coefs, '__iter__'): + flow_rate_coefs = [flow_rate_coefs] + if not hasattr(departSpeeds, '__iter__'): + departSpeeds = [departSpeeds] + + for departSpeed in departSpeeds: + for penetration_rate in penetration_rates: + for flow_rate_coef in flow_rate_coefs: + yield InflowTransfer(penetration_rate=penetration_rate, flow_rate_coef=flow_rate_coef, + departSpeed=departSpeed) diff --git a/flow/visualize/visualize_fuel_consumption.py b/flow/visualize/visualize_fuel_consumption.py new file mode 100644 index 000000000..f1f5c05be --- /dev/null +++ b/flow/visualize/visualize_fuel_consumption.py @@ -0,0 +1,144 @@ +"""Script to visualize the fuel consumption x speed x accel trajectory.""" +from matplotlib import pyplot as plt +from mpl_toolkits.mplot3d import Axes3D +import pandas as pd +import argparse +import os +import os.path + +try: + import imageio +except ImportError: + print('Please install the imageio library (eg. `pip install imageio`)') + exit(0) + +import flow.config as config +from flow.energy_models.power_demand import PDMCombustionEngine + +EXAMPLE_USAGE = """ +example usage: + python ./visualize_energy /path/to/emissions.csv +""" + + +def generate_fuel_graph(args): + """Generate the trajectory graph.""" + sim_step = args.sim_step + verif_path = os.path.join(config.PROJECT_PATH, + 'flow/visualize/energy_model_verification.csv') + plot_all_time_steps = args.plot_all_time_steps + step_interval = args.step_interval + + azim = 225 + verif = pd.read_csv(verif_path) + fig = plt.figure(figsize=(10, 10)) + fig.gca(projection=Axes3D.name) + ax1 = fig.add_subplot(1, 1, 1, projection='3d') + ax1.plot_trisurf(verif['speed(m/s)'], + verif['acceleration(m/s^2)'], + verif['Tacoma_fit(gal/hr)'] * sim_step / 3600.0, + linewidth=0.2) + ax1.set_title('Tacoma-fit Fuel Consumption', fontsize=18) + ax1.set_xlabel('Speed (m/s)', fontsize=14) + ax1.set_ylabel('Acceleration (m/s/s)', fontsize=14) + ax1.set_zlabel('Fuel Rate', fontsize=14) + ax1.set_zticks([]) + ax1.view_init(azim=azim) + energy_model = PDMCombustionEngine() + + baseline = energy_model.get_instantaneous_fuel_consumption(0, 12, 0) + baseline = baseline * sim_step / 3600.0 + ax1.plot([12], [0], [baseline], + marker='o', markersize=4, color='#5AFE3E', zorder=20) + + filename = args.emission_file + df = pd.read_csv(filename) + sim_step = sorted(df['time'].unique())[1] - sorted(df['time'].unique())[0] + df.set_index('time', inplace=True) + df['fuel'] = df.apply(lambda x: energy_model.get_instantaneous_fuel_consumption( + x['target_accel_no_noise_with_failsafe'], x['speed'], 0), axis=1) + + png_files = [] + for i, row in df.iterrows(): + if (plot_all_time_steps or i == int(i)) and i % step_interval == 0: + print(i) + first = max(i-1.0, 0.0) + p1 = ax1.plot(df.loc[first:i]['speed'], + df.loc[first:i]['target_accel_no_noise_with_failsafe'], + df.loc[first:i]['fuel'] * sim_step / 3600.0, + linewidth=1, color='r', zorder=20) + p2 = ax1.plot([row['speed']], + [row['target_accel_no_noise_with_failsafe']], + [row['fuel'] * sim_step / 3600.0], + marker='o', markersize=4, color='y', zorder=20) + t = ax1.annotate('Time: {} s'.format(i), (0.2, 0.9), + xycoords='figure fraction', fontsize=20) + png_name = 'traj_{}.png'.format(i) + plt.savefig(png_name) + png_files.append(png_name) + p1.pop(0).remove() + p2.pop(0).remove() + t.remove() + print() + + if not args.no_gif: + out_path = args.emission_file.replace('csv', 'gif') + with imageio.get_writer(out_path, mode='I') as writer: + for filename in png_files: + image = imageio.imread(filename) + writer.append_data(image) + print('Generate fuel visualization at ' + out_path) + + for filename in png_files: + os.remove(filename) + + +def create_parser(): + """Create the parser to capture CLI arguments.""" + parser = argparse.ArgumentParser( + description='[Flow] Visualize trajectory of agent on speed x ' + 'acceleration x fuel rate graph.', + epilog=EXAMPLE_USAGE) + + # required input parameters + parser.add_argument( + 'emission_file', + type=str, + help='Path to emission .csv file') + + # optional input parameters + parser.add_argument( + '--sim_step', + type=float, + default=0.1, + help='The sim step used when generating the emissions.') + parser.add_argument( + '--plot_all_time_steps', + type=bool, + default=False, + help='Specifies whether to visualize all time steps or just the first ' + 'one. For instance, if sims_per_step is 5 and this is set, all ' + '5 time steps of each simulation step will be visualized.') + parser.add_argument( + '--step_interval', + type=int, + default=1, + help='Specifies n where every n simulation steps will be visualized.') + parser.add_argument( + '--no_gif', + type=bool, + default=False, + help='If this is set, the png will not be turned into a gif.') + + return parser + + +def main(): + """Generate the fuel consumption graph.""" + parser = create_parser() + args = parser.parse_args() + generate_fuel_graph(args) + + +if __name__ == '__main__': + main() diff --git a/flow/visualize/visualizer_rllib.py b/flow/visualize/visualizer_rllib.py deleted file mode 100644 index c1dd83193..000000000 --- a/flow/visualize/visualizer_rllib.py +++ /dev/null @@ -1,386 +0,0 @@ -"""Visualizer for rllib experiments. - -Attributes ----------- -EXAMPLE_USAGE : str - Example call to the function, which is - :: - - python ./visualizer_rllib.py /tmp/ray/result_dir 1 - -parser : ArgumentParser - Command-line argument parser -""" - -import argparse -import gym -import numpy as np -import os -import sys -import time - -import ray -try: - from ray.rllib.agents.agent import get_agent_class -except ImportError: - from ray.rllib.agents.registry import get_agent_class -from ray.tune.registry import register_env - -from flow.core.util import emission_to_csv -from flow.utils.registry import make_create_env -from flow.utils.rllib import get_flow_params -from flow.utils.rllib import get_rllib_config -from flow.utils.rllib import get_rllib_pkl - - -EXAMPLE_USAGE = """ -example usage: - python ./visualizer_rllib.py /ray_results/experiment_dir/result_dir 1 - -Here the arguments are: -1 - the path to the simulation results -2 - the number of the checkpoint -""" - - -def visualizer_rllib(args): - """Visualizer for RLlib experiments. - - This function takes args (see function create_parser below for - more detailed information on what information can be fed to this - visualizer), and renders the experiment associated with it. - """ - result_dir = args.result_dir if args.result_dir[-1] != '/' \ - else args.result_dir[:-1] - - config = get_rllib_config(result_dir) - - # check if we have a multiagent environment but in a - # backwards compatible way - if config.get('multiagent', {}).get('policies', None): - multiagent = True - pkl = get_rllib_pkl(result_dir) - config['multiagent'] = pkl['multiagent'] - else: - multiagent = False - - # Run on only one cpu for rendering purposes - config['num_workers'] = 0 - - flow_params = get_flow_params(config) - - # hack for old pkl files - # TODO(ev) remove eventually - sim_params = flow_params['sim'] - setattr(sim_params, 'num_clients', 1) - - # for hacks for old pkl files TODO: remove eventually - if not hasattr(sim_params, 'use_ballistic'): - sim_params.use_ballistic = False - - # Determine agent and checkpoint - config_run = config['env_config']['run'] if 'run' in config['env_config'] \ - else None - if args.run and config_run: - if args.run != config_run: - print('visualizer_rllib.py: error: run argument ' - + '\'{}\' passed in '.format(args.run) - + 'differs from the one stored in params.json ' - + '\'{}\''.format(config_run)) - sys.exit(1) - if args.run: - agent_cls = get_agent_class(args.run) - elif config_run: - agent_cls = get_agent_class(config_run) - else: - print('visualizer_rllib.py: error: could not find flow parameter ' - '\'run\' in params.json, ' - 'add argument --run to provide the algorithm or model used ' - 'to train the results\n e.g. ' - 'python ./visualizer_rllib.py /tmp/ray/result_dir 1 --run PPO') - sys.exit(1) - - sim_params.restart_instance = True - dir_path = os.path.dirname(os.path.realpath(__file__)) - emission_path = '{0}/test_time_rollout/'.format(dir_path) - sim_params.emission_path = emission_path if args.gen_emission else None - - # pick your rendering mode - if args.render_mode == 'sumo_web3d': - sim_params.num_clients = 2 - sim_params.render = False - elif args.render_mode == 'drgb': - sim_params.render = 'drgb' - sim_params.pxpm = 4 - elif args.render_mode == 'sumo_gui': - sim_params.render = False # will be set to True below - elif args.render_mode == 'no_render': - sim_params.render = False - if args.save_render: - if args.render_mode != 'sumo_gui': - sim_params.render = 'drgb' - sim_params.pxpm = 4 - sim_params.save_render = True - - # Create and register a gym+rllib env - create_env, env_name = make_create_env(params=flow_params, version=0) - register_env(env_name, create_env) - - # check if the environment is a single or multiagent environment, and - # get the right address accordingly - # single_agent_envs = [env for env in dir(flow.envs) - # if not env.startswith('__')] - - # if flow_params['env_name'] in single_agent_envs: - # env_loc = 'flow.envs' - # else: - # env_loc = 'flow.envs.multiagent' - - # Start the environment with the gui turned on and a path for the - # emission file - env_params = flow_params['env'] - env_params.restart_instance = False - if args.evaluate: - env_params.evaluate = True - - # lower the horizon if testing - if args.horizon: - config['horizon'] = args.horizon - env_params.horizon = args.horizon - - # create the agent that will be used to compute the actions - agent = agent_cls(env=env_name, config=config) - checkpoint = result_dir + '/checkpoint_' + args.checkpoint_num - checkpoint = checkpoint + '/checkpoint-' + args.checkpoint_num - agent.restore(checkpoint) - - if hasattr(agent, "local_evaluator") and \ - os.environ.get("TEST_FLAG") != 'True': - env = agent.local_evaluator.env - else: - env = gym.make(env_name) - - if args.render_mode == 'sumo_gui': - env.sim_params.render = True # set to True after initializing agent and env - - if multiagent: - rets = {} - # map the agent id to its policy - policy_map_fn = config['multiagent']['policy_mapping_fn'] - for key in config['multiagent']['policies'].keys(): - rets[key] = [] - else: - rets = [] - - if config['model']['use_lstm']: - use_lstm = True - if multiagent: - state_init = {} - # map the agent id to its policy - policy_map_fn = config['multiagent']['policy_mapping_fn'] - size = config['model']['lstm_cell_size'] - for key in config['multiagent']['policies'].keys(): - state_init[key] = [np.zeros(size, np.float32), - np.zeros(size, np.float32)] - else: - state_init = [ - np.zeros(config['model']['lstm_cell_size'], np.float32), - np.zeros(config['model']['lstm_cell_size'], np.float32) - ] - else: - use_lstm = False - - # if restart_instance, don't restart here because env.reset will restart later - if not sim_params.restart_instance: - env.restart_simulation(sim_params=sim_params, render=sim_params.render) - - # Simulate and collect metrics - final_outflows = [] - final_inflows = [] - mean_speed = [] - std_speed = [] - for i in range(args.num_rollouts): - vel = [] - state = env.reset() - if multiagent: - ret = {key: [0] for key in rets.keys()} - else: - ret = 0 - for _ in range(env_params.horizon): - vehicles = env.unwrapped.k.vehicle - speeds = vehicles.get_speed(vehicles.get_ids()) - - # only include non-empty speeds - if speeds: - vel.append(np.mean(speeds)) - - if multiagent: - action = {} - for agent_id in state.keys(): - if use_lstm: - action[agent_id], state_init[agent_id], logits = \ - agent.compute_action( - state[agent_id], state=state_init[agent_id], - policy_id=policy_map_fn(agent_id)) - else: - action[agent_id] = agent.compute_action( - state[agent_id], policy_id=policy_map_fn(agent_id)) - else: - action = agent.compute_action(state) - state, reward, done, _ = env.step(action) - if multiagent: - for actor, rew in reward.items(): - ret[policy_map_fn(actor)][0] += rew - else: - ret += reward - if multiagent and done['__all__']: - break - if not multiagent and done: - break - - if multiagent: - for key in rets.keys(): - rets[key].append(ret[key]) - else: - rets.append(ret) - outflow = vehicles.get_outflow_rate(500) - final_outflows.append(outflow) - inflow = vehicles.get_inflow_rate(500) - final_inflows.append(inflow) - if np.all(np.array(final_inflows) > 1e-5): - throughput_efficiency = [x / y for x, y in - zip(final_outflows, final_inflows)] - else: - throughput_efficiency = [0] * len(final_inflows) - mean_speed.append(np.mean(vel)) - std_speed.append(np.std(vel)) - if multiagent: - for agent_id, rew in rets.items(): - print('Round {}, Return: {} for agent {}'.format( - i, ret, agent_id)) - else: - print('Round {}, Return: {}'.format(i, ret)) - - print('==== Summary of results ====') - print("Return:") - print(mean_speed) - if multiagent: - for agent_id, rew in rets.items(): - print('For agent', agent_id) - print(rew) - print('Average, std return: {}, {} for agent {}'.format( - np.mean(rew), np.std(rew), agent_id)) - else: - print(rets) - print('Average, std: {}, {}'.format( - np.mean(rets), np.std(rets))) - - print("\nSpeed, mean (m/s):") - print(mean_speed) - print('Average, std: {}, {}'.format(np.mean(mean_speed), np.std( - mean_speed))) - print("\nSpeed, std (m/s):") - print(std_speed) - print('Average, std: {}, {}'.format(np.mean(std_speed), np.std( - std_speed))) - - # Compute arrival rate of vehicles in the last 500 sec of the run - print("\nOutflows (veh/hr):") - print(final_outflows) - print('Average, std: {}, {}'.format(np.mean(final_outflows), - np.std(final_outflows))) - # Compute departure rate of vehicles in the last 500 sec of the run - print("Inflows (veh/hr):") - print(final_inflows) - print('Average, std: {}, {}'.format(np.mean(final_inflows), - np.std(final_inflows))) - # Compute throughput efficiency in the last 500 sec of the - print("Throughput efficiency (veh/hr):") - print(throughput_efficiency) - print('Average, std: {}, {}'.format(np.mean(throughput_efficiency), - np.std(throughput_efficiency))) - - # terminate the environment - env.unwrapped.terminate() - - # if prompted, convert the emission file into a csv file - if args.gen_emission: - time.sleep(0.1) - - dir_path = os.path.dirname(os.path.realpath(__file__)) - emission_filename = '{0}-emission.xml'.format(env.network.name) - - emission_path = \ - '{0}/test_time_rollout/{1}'.format(dir_path, emission_filename) - - # convert the emission file into a csv file - emission_to_csv(emission_path) - - # print the location of the emission csv file - emission_path_csv = emission_path[:-4] + ".csv" - print("\nGenerated emission file at " + emission_path_csv) - - # delete the .xml version of the emission file - os.remove(emission_path) - - -def create_parser(): - """Create the parser to capture CLI arguments.""" - parser = argparse.ArgumentParser( - formatter_class=argparse.RawDescriptionHelpFormatter, - description='[Flow] Evaluates a reinforcement learning agent ' - 'given a checkpoint.', - epilog=EXAMPLE_USAGE) - - # required input parameters - parser.add_argument( - 'result_dir', type=str, help='Directory containing results') - parser.add_argument('checkpoint_num', type=str, help='Checkpoint number.') - - # optional input parameters - parser.add_argument( - '--run', - type=str, - help='The algorithm or model to train. This may refer to ' - 'the name of a built-on algorithm (e.g. RLLib\'s DQN ' - 'or PPO), or a user-defined trainable function or ' - 'class registered in the tune registry. ' - 'Required for results trained with flow-0.2.0 and before.') - parser.add_argument( - '--num_rollouts', - type=int, - default=1, - help='The number of rollouts to visualize.') - parser.add_argument( - '--gen_emission', - action='store_true', - help='Specifies whether to generate an emission file from the ' - 'simulation') - parser.add_argument( - '--evaluate', - action='store_true', - help='Specifies whether to use the \'evaluate\' reward ' - 'for the environment.') - parser.add_argument( - '--render_mode', - type=str, - default='sumo_gui', - help='Pick the render mode. Options include sumo_web3d, ' - 'rgbd and sumo_gui') - parser.add_argument( - '--save_render', - action='store_true', - help='Saves a rendered video to a file. NOTE: Overrides render_mode ' - 'with pyglet rendering.') - parser.add_argument( - '--horizon', - type=int, - help='Specifies the horizon.') - return parser - - -if __name__ == '__main__': - parser = create_parser() - args = parser.parse_args() - ray.init(num_cpus=1) - visualizer_rllib(args) diff --git a/requirements.txt b/requirements.txt index a4f6f83f8..0d1ddbf83 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ pyglet==1.3.2 matplotlib==3.1.0 imutils==0.5.1 numpydoc -ray==0.8.0 +ray==0.8.4 opencv-python dill lz4 @@ -22,7 +22,7 @@ psutil opencv-python boto3==1.10.45 redis~=2.10.6 -pandas==0.24.2 +pandas==1.1.2 plotly==2.4.0 tabulate tensorflow==1.15.2 diff --git a/scripts/ray_autoscale.yaml b/scripts/ray_autoscale.yaml index 27ac0898e..4d179ff2b 100644 --- a/scripts/ray_autoscale.yaml +++ b/scripts/ray_autoscale.yaml @@ -1,4 +1,4 @@ -# cluster.yaml ========================================= +# cluster.yaml ========================================= # An unique identifier for the head node and workers of this cluster. cluster_name: test # @@ -32,15 +32,15 @@ auth: # By default Ray creates a new private keypair, but you can also use your own. # If you do so, make sure to also set "KeyName" in the head and worker node # configurations below. -# ssh_private_key: /path/to/your/key.pem +# ssh_private_key: # Provider-specific config for the head node, e.g. instance type. By default # Ray will auto-configure unspecified fields such as SubnetId and KeyName. # For more documentation on available fields, see: # http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.ServiceResource.create_instances head_node: - InstanceType: c4.4xlarge - ImageId: ami-09544298704576518 # Flow AMI (Ubuntu) + InstanceType: c4.8xlarge + ImageId: ami-03d50e10b75b75bd5 # circles_2 AMI (Ubuntu) InstanceMarketOptions: MarketType: spot #Additional options can be found in the boto docs, e.g. @@ -54,10 +54,10 @@ head_node: # For more documentation on available fields, see: # http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.ServiceResource.create_instances worker_nodes: - InstanceType: c4.4xlarge - ImageId: ami-09544298704576518 # Flow AMI (Ubuntu) + InstanceType: c4.8xlarge + ImageId: ami-03d50e10b75b75bd5 # circles_2 AMI (Ubuntu) - #Run workers on spot by default. Comment this out to use on-demand. + #Run workers on spot by default. Comment this out to use on-demand. InstanceMarketOptions: MarketType: spot # Additional options can be found in the boto docs, e.g. @@ -67,12 +67,19 @@ worker_nodes: # Additional options in the boto docs. setup_commands: - - cd flow && git fetch && git checkout origin/master + - cd flow && git fetch && git checkout origin/flow_maddpg + - flow/scripts/setup_sumo_ubuntu1604.sh + - pip install ray==0.8.0 + - pip install tabulate - pip install boto3==1.10.45 # 1.4.8 adds InstanceMarketOptions - pip install awscli==1.16.309 + - pip install stable-baselines - pip install pytz + - pip install torch==1.3.1 + - pip install tensorflow==2.0.0 + - pip install lz4 + - pip install dm-tree - pip install numpy==1.18.4 - - ./flow/scripts/setup_sumo_ubuntu1604.sh head_setup_commands: [] diff --git a/scripts/setI210Length.sh b/scripts/setI210Length.sh new file mode 100755 index 000000000..f9bccaa36 --- /dev/null +++ b/scripts/setI210Length.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# run this like so: ./setI210Length.sh "/path/to/flow/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml" DESIRED_LENGTH + +# Argument parsing + +XML_FILE=$1 # path to the xml file +LENGTH=$2 # desired length + +# Some constants + +X0_EDGE="2148.42" # original starting x value for the edge +Y0_EDGE="1244.27" # original starting y value for the edge +X0_LANE0="2155.88" # original starting x value for the first lane +Y0_LANE0="1239.42" # original starting y value for the first lane +X0_LANE1="2154.23" # original starting x value for the second lane +Y0_LANE1="1242.16" # original starting y value for the second lane +X0_LANE2="2152.58" # original starting x value for the third lane +Y0_LANE2="1244.91" # original starting y value for the third lane +X0_LANE3="2150.94" # original starting x value for the fourth lane +Y0_LANE3="1247.65" # original starting y value for the fourth lane +X0_LANE4="2149.29" # original starting x value for the fifth lane +Y0_LANE4="1250.39" # original starting y value for the fifth lane +X0_LANE5="2147.64" # original starting x value for the sixth lane +Y0_LANE5="1253.13" # original starting y value for the sixth lane +DX="0.85" # original (x2-x1) / length +DY="0.52" # original (y2-y1) / length + +# Parsing operation. + +X1_EDGE=$(bc <<< "scale=2; $X0_EDGE + $LENGTH * $DX") +Y1_EDGE=$(bc <<< "scale=2; $Y0_EDGE + $LENGTH * $DY") +X1_LANE0=$(bc <<< "scale=2; $X0_LANE0 + $LENGTH * $DX") +Y1_LANE0=$(bc <<< "scale=2; $Y0_LANE0 + $LENGTH * $DY") +X1_LANE1=$(bc <<< "scale=2; $X0_LANE1 + $LENGTH * $DX") +Y1_LANE1=$(bc <<< "scale=2; $Y0_LANE1 + $LENGTH * $DY") +X1_LANE2=$(bc <<< "scale=2; $X0_LANE2 + $LENGTH * $DX") +Y1_LANE2=$(bc <<< "scale=2; $Y0_LANE2 + $LENGTH * $DY") +X1_LANE3=$(bc <<< "scale=2; $X0_LANE3 + $LENGTH * $DX") +Y1_LANE3=$(bc <<< "scale=2; $Y0_LANE3 + $LENGTH * $DY") +X1_LANE4=$(bc <<< "scale=2; $X0_LANE4 + $LENGTH * $DX") +Y1_LANE4=$(bc <<< "scale=2; $Y0_LANE4 + $LENGTH * $DY") +X1_LANE5=$(bc <<< "scale=2; $X0_LANE5 + $LENGTH * $DX") +Y1_LANE5=$(bc <<< "scale=2; $Y0_LANE5 + $LENGTH * $DY") + +OLD_EDGE=" " +NEW_LANE0=" " +NEW_LANE1=" " +NEW_LANE2=" " +NEW_LANE3=" " +NEW_LANE4=" " +NEW_LANE5=" " + +sed -i '' '/'"${OLD_EDGE}"'/s/.*/'"${NEW_EDGE}"'/' "${XML_FILE}" +sed -i '' '/'"${OLD_LANE0}"'/s/.*/'"${NEW_LANE0}"'/' "${XML_FILE}" +sed -i '' '/'"${OLD_LANE1}"'/s/.*/'"${NEW_LANE1}"'/' "${XML_FILE}" +sed -i '' '/'"${OLD_LANE2}"'/s/.*/'"${NEW_LANE2}"'/' "${XML_FILE}" +sed -i '' '/'"${OLD_LANE3}"'/s/.*/'"${NEW_LANE3}"'/' "${XML_FILE}" +sed -i '' '/'"${OLD_LANE4}"'/s/.*/'"${NEW_LANE4}"'/' "${XML_FILE}" +sed -i '' '/'"${OLD_LANE5}"'/s/.*/'"${NEW_LANE5}"'/' "${XML_FILE}" \ No newline at end of file diff --git a/tests/data/rllib_data/multi_agent/checkpoint_1/.is_checkpoint b/tests/data/rllib_data/multi_agent/checkpoint_1/.is_checkpoint new file mode 100644 index 000000000..e69de29bb diff --git a/tests/data/rllib_data/multi_agent/checkpoint_1/checkpoint-1 b/tests/data/rllib_data/multi_agent/checkpoint_1/checkpoint-1 index d346e9dc5..127111e67 100644 Binary files a/tests/data/rllib_data/multi_agent/checkpoint_1/checkpoint-1 and b/tests/data/rllib_data/multi_agent/checkpoint_1/checkpoint-1 differ diff --git a/tests/data/rllib_data/multi_agent/checkpoint_1/checkpoint-1.tune_metadata b/tests/data/rllib_data/multi_agent/checkpoint_1/checkpoint-1.tune_metadata index febe7b205..268e5cfe5 100644 Binary files a/tests/data/rllib_data/multi_agent/checkpoint_1/checkpoint-1.tune_metadata and b/tests/data/rllib_data/multi_agent/checkpoint_1/checkpoint-1.tune_metadata differ diff --git a/tests/data/rllib_data/multi_agent/params.json b/tests/data/rllib_data/multi_agent/params.json index 1aa9a114c..b5bf22611 100644 --- a/tests/data/rllib_data/multi_agent/params.json +++ b/tests/data/rllib_data/multi_agent/params.json @@ -1,35 +1,40 @@ { "batch_mode": "truncate_episodes", "callbacks": { - "on_episode_end": null, - "on_episode_start": null, - "on_episode_step": null, - "on_postprocess_traj": null, - "on_sample_end": null, - "on_train_result": null + "on_episode_end": ".on_episode_end at 0x7fdfc4d70488>", + "on_episode_start": ".on_episode_start at 0x7fdf8056e840>", + "on_episode_step": ".on_episode_step at 0x7fdfc4d68400>", + "on_train_result": ".on_train_result at 0x7fdfc4d70510>" }, "clip_actions": true, "clip_param": 0.3, "clip_rewards": null, "collect_metrics_timeout": 180, "compress_observations": false, + "custom_eval_function": null, "custom_resources_per_worker": {}, "eager": false, "eager_tracing": false, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, - "env": "MultiAgentAccelPOEnv-v1", + "env": "MultiAgentWaveAttenuationPOEnv-v1", "env_config": { - "flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 3,\n \"max_decel\": 3,\n \"sort_vehicles\": false,\n \"target_velocity\": 20\n },\n \"clip_actions\": true,\n \"evaluate\": false,\n \"horizon\": 1500,\n \"sims_per_step\": 1,\n \"warmup_steps\": 0\n },\n \"env_name\": \"flow.envs.multiagent.ring.accel.MultiAgentAccelPOEnv\",\n \"exp_tag\": \"multiagent_figure_eight\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"radius_ring\": 30,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.figure_eight.FigureEightNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1,\n \"use_ballistic\": false\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 1.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 1\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 6,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human_0\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 3,\n \"carFollowModel\": \"IDM\",\n \"decel\": 3,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 1\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl_0\"\n },\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 1.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 1\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 6,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human_1\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 3,\n \"carFollowModel\": \"IDM\",\n \"decel\": 3,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 1\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl_1\"\n }\n ]\n}", - "run": "PPO" + "flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.multiagent.ring.wave_attenuation.MultiAgentWaveAttenuationPOEnv\",\n \"exp_tag\": \"multiagent_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": false,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"load_state\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"save_state_file\": \"initial_state.xml\",\n \"save_state_time\": null,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1,\n \"use_ballistic\": false\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl_0\"\n },\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 10,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human_0\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl_1\"\n },\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 10,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human_1\"\n }\n ]\n}", + "run": "" }, "evaluation_config": {}, "evaluation_interval": null, "evaluation_num_episodes": 10, - "gamma": 0.999, + "evaluation_num_workers": 0, + "exploration_config": { + "type": "StochasticSampling" + }, + "explore": true, + "gamma": 0.995, "grad_clip": null, - "horizon": 1500, + "horizon": 3000, "ignore_worker_failures": false, + "in_evaluation": false, "input": "sampler", "input_evaluation": [ "is", @@ -60,7 +65,6 @@ "dim": 84, "fcnet_activation": "tanh", "fcnet_hiddens": [ - 32, 32, 32 ], @@ -81,23 +85,24 @@ "policies": { "av": [ "", - "Box(6,)", + "Box(3,)", "Box(1,)", {} ] }, "policies_to_train": null, - "policy_mapping_fn": "" + "policy_mapping_fn": "" }, - "no_done_at_end": false, + "no_done_at_end": true, "no_eager_on_workers": false, + "normalize_actions": false, "num_cpus_for_driver": 1, "num_cpus_per_worker": 1, "num_envs_per_worker": 1, "num_gpus": 0, "num_gpus_per_worker": 0, "num_sgd_iter": 10, - "num_workers": 2, + "num_workers": 1, "object_store_memory": 0, "object_store_memory_per_worker": 0, "observation_filter": "NoFilter", @@ -112,8 +117,9 @@ "preprocessor_pref": "deepmind", "remote_env_batch_wait_ms": 0, "remote_worker_envs": false, + "rollout_fragment_length": 200, "sample_async": false, - "sample_batch_size": 200, + "sample_batch_size": -1, "seed": null, "sgd_minibatch_size": 128, "shuffle_buffer_size": 0, @@ -134,8 +140,11 @@ "log_device_placement": false }, "timesteps_per_iteration": 0, - "train_batch_size": 30000, + "train_batch_size": 3000, + "use_critic": true, + "use_exec_api": false, "use_gae": true, + "use_pytorch": false, "vf_clip_param": 10.0, "vf_loss_coeff": 1.0, "vf_share_layers": false diff --git a/tests/data/rllib_data/multi_agent/params.pkl b/tests/data/rllib_data/multi_agent/params.pkl index 192cf7558..d1260bcbb 100644 Binary files a/tests/data/rllib_data/multi_agent/params.pkl and b/tests/data/rllib_data/multi_agent/params.pkl differ diff --git a/tests/data/rllib_data/single_agent/params.json b/tests/data/rllib_data/single_agent/params.json index c93580225..e4f8732c3 100644 --- a/tests/data/rllib_data/single_agent/params.json +++ b/tests/data/rllib_data/single_agent/params.json @@ -20,8 +20,8 @@ "entropy_coeff_schedule": null, "env": "AccelEnv-v0", "env_config": { - "flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 3,\n \"max_decel\": 3,\n \"sort_vehicles\": false,\n \"target_velocity\": 20\n },\n \"clip_actions\": true,\n \"evaluate\": false,\n \"horizon\": 1500,\n \"sims_per_step\": 1,\n \"warmup_steps\": 0\n },\n \"env_name\": \"flow.envs.ring.accel.AccelEnv\",\n \"exp_tag\": \"singleagent_figure_eight\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"radius_ring\": 30,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.figure_eight.FigureEightNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1,\n \"use_ballistic\": false\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 1.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 1\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 13,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 1.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 1\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", - "run": "PPO" + "run": "PPO", + "flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 3,\n \"max_decel\": 3,\n \"sort_vehicles\": false,\n \"target_velocity\": 20\n },\n \"clip_actions\": true,\n \"evaluate\": false,\n \"horizon\": 1500,\n \"sims_per_step\": 1,\n \"warmup_steps\": 0\n },\n \"env_name\": \"flow.envs.ring.accel.AccelEnv\",\n \"exp_tag\": \"singleagent_figure_eight\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"radius_ring\": 30,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.figure_eight.FigureEightNetwork\",\n \"sim\": {\n \"save_state_time\": null,\n \"save_state_file\": \"initial_state.xml\",\n \"load_state\": null,\n \"disable_collisions\": false,\n \"color_by_speed\": false,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1,\n \"use_ballistic\": false\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 1.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 1\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 13,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 1.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 1\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}" }, "evaluation_config": {}, "evaluation_interval": null, diff --git a/tests/fast_tests/test_energy_models.py b/tests/fast_tests/test_energy_models.py new file mode 100644 index 000000000..a1724e38b --- /dev/null +++ b/tests/fast_tests/test_energy_models.py @@ -0,0 +1,45 @@ +import unittest +import os +import pandas as pd + +from pandas.testing import assert_series_equal + +from flow.energy_models.power_demand import PDMCombustionEngine, PDMElectric + +os.environ["TEST_FLAG"] = "True" + + +class TestEnergyModels(unittest.TestCase): + """ + Tests all energy models used in flow + """ + + def setUp(self): + """ + Load verification data from csv + """ + dir_path = os.path.dirname(os.path.realpath(__file__)) + self.data = pd.read_csv(os.path.join(dir_path, "test_files/energy_model_verification_data.csv")) + print(self.data) + + def test_pdm_combustion_engine_power(self): + energy_model = PDMCombustionEngine() + power_value = energy_model.get_instantaneous_power + power = self.data.apply(lambda x: power_value(x['a(m/s^2)'], x['v(m/s)'], x['rg(%)']), axis=1) + assert_series_equal(self.data['Tacoma_fit(KW)']*1000, power, rtol=0.1, check_names=False) + + def test_pdm_combustion_engine_fuel(self): + energy_model = PDMCombustionEngine() + fuel_value = energy_model.get_instantaneous_fuel_consumption + fuel = self.data.apply(lambda x: fuel_value(x['a(m/s^2)'], x['v(m/s)'], x['rg(%)']), axis=1) + assert_series_equal(self.data['Tacoma_fit(gal/hr)'], fuel, rtol=0.1, check_names=False) + + def test_pdm_electric_power(self): + energy_model = PDMElectric() + power_value = energy_model.get_instantaneous_power + power = self.data.apply(lambda x: power_value(x['a(m/s^2)'], x['v(m/s)'], x['rg(%)']), axis=1) + assert_series_equal(self.data['Prius_fit(KW)']*1000, power, rtol=0.1, check_names=False) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/fast_tests/test_environment_base_class.py b/tests/fast_tests/test_environment_base_class.py index ee815393c..2d391755a 100644 --- a/tests/fast_tests/test_environment_base_class.py +++ b/tests/fast_tests/test_environment_base_class.py @@ -10,6 +10,11 @@ from flow.envs.ring.accel import ADDITIONAL_ENV_PARAMS from flow.utils.exceptions import FatalFlowError from flow.envs import Env, TestEnv +from flow.envs.multiagent import MultiAgentAccelPOEnv +from flow.networks import RingNetwork +from flow.core.experiment import Experiment +from flow.utils.registry import make_create_env +import flow.config as config from tests.setup_scripts import ring_road_exp_setup, highway_exp_setup import os @@ -541,5 +546,164 @@ def test_clip_tuple_actions(self): self.assertEquals(_actions[2], clipped_actions[2]) +class TestSaveLoadState(unittest.TestCase): + """Tests for the save and load state features for sumo.""" + + def setUp(self): + sim_params = SumoParams(sim_step=0.1, render=False) + + vehicles = VehicleParams() + vehicles.add( + veh_id="human", + acceleration_controller=(IDMController, {}), + routing_controller=(ContinuousRouter, {}), + car_following_params=SumoCarFollowingParams( + speed_mode="aggressive", + ), + num_vehicles=1, + ) + + env_params = EnvParams( + horizon=20, + additional_params={ + "target_velocity": 8, + "max_accel": 1, + "max_decel": 1, + "sort_vehicles": False, + } + ) + + net_params = NetParams(additional_params={ + "length": 230, + "lanes": 1, + "speed_limit": 30, + "resolution": 40 + }) + + self.flow_params = dict( + exp_tag="RingRoadTest", + env_name=TestEnv, + network=RingNetwork, + simulator='traci', + sim=sim_params, + env=env_params, + net=net_params, + veh=vehicles, + ) + + def tearDown(self): + del self.flow_params + + def test_save_state(self): + """Test the save_state option. + + This verifies that: + + 1. the xml file is generated + 2. the starting time is as expected. + """ + # Add the save option. + save_state_file = "test_save_state.xml" + self.flow_params['sim'].save_state_time = 1 + self.flow_params['sim'].save_state_file = save_state_file + + # Run the simulation. + exp = Experiment(self.flow_params) + exp.run(num_runs=1) + + # test case 1 + self.assertTrue(os.path.isfile(save_state_file)) + + # test case 2 + read_time = 0 + read_file = "" + with open(save_state_file, 'r') as f: + # Read all lines in the file one by one + for line in f: + # For each line, check if line contains the string + if "save-state.times" in line: + read_time = float(line.split("\"")[1]) + elif "save-state.files" in line: + read_file = line.split("\"")[1] + self.assertAlmostEqual(read_time, 1.0) + self.assertEqual(read_file, save_state_file) + + # Remove the generated file. + os.remove(save_state_file) + + def test_load_state_singleagent(self): + """Test the load_state option for single-agent environments. + + This verifies that the number and initial states of vehicles are + correctly loaded from the xml file. + """ + # Add the load option. + self.flow_params['sim'].load_state = os.path.join( + config.PROJECT_PATH, + "tests/fast_tests/test_files/test_save_state.xml") + + # Run the reset option. + create_env, _ = make_create_env(self.flow_params) + env = create_env() + env.reset() + + # Run assertions. + self.assertEqual( + env.k.vehicle.num_vehicles, 10) + self.assertEqual( + env.k.vehicle.get_ids(), ["human_{}".format(i) for i in range(10)]) + np.testing.assert_almost_equal( + env.k.vehicle.get_speed(env.k.vehicle.get_ids()), + [1.281359, 1.2813589, 1.281359, 1.2813591, 1.281693, 1.4292893, + 1.281359, 1.2813587, 1.2813592, 1.2821579]) + np.testing.assert_almost_equal( + env.k.vehicle.get_x_by_id(env.k.vehicle.get_ids()), + [0.5812726, 23.5812725, 46.5812726, 69.5812726, 92.5813197, + 115.6609682, 138.5812726, 161.5812725, 184.5812726, 207.5813896]) + np.testing.assert_almost_equal( + [env.k.vehicle.get_distance(veh_id) + for veh_id in env.k.vehicle.get_ids()], + [0.5812726, 0.5812725, 0.5812726, 0.5812726, 0.5813197, 0.6609682, + 0.5812726, 0.5812725, 0.5812726, 0.5813896]) + + def test_load_state_multiagent(self): + """Test the load_state option for multi-agent environments. + + This verifies that the number and initial states of vehicles are + correctly loaded from the xml file. + """ + # Add the load option. + self.flow_params['sim'].load_state = os.path.join( + config.PROJECT_PATH, + "tests/fast_tests/test_files/test_save_state.xml") + + # Switch to multiagent environment. + self.flow_params['env_name'] = MultiAgentAccelPOEnv + + # Run the reset option. + create_env, _ = make_create_env(self.flow_params) + env = create_env() + env.reset() + + # Run assertions. + self.assertEqual( + env.k.vehicle.num_vehicles, 10) + self.assertEqual( + env.k.vehicle.get_ids(), ["human_{}".format(i) for i in range(10)]) + np.testing.assert_almost_equal( + env.k.vehicle.get_speed(env.k.vehicle.get_ids()), + [1.281359, 1.2813589, 1.281359, 1.2813591, 1.281693, 1.4292893, + 1.281359, 1.2813587, 1.2813592, 1.2821579]) + np.testing.assert_almost_equal( + env.k.vehicle.get_x_by_id(env.k.vehicle.get_ids()), + [0.5812726, 23.5812725, 46.5812726, 69.5812726, 92.5813197, + 115.6609682, 138.5812726, 161.5812725, 184.5812726, 207.5813896]) + np.testing.assert_almost_equal( + [env.k.vehicle.get_distance(veh_id) + for veh_id in env.k.vehicle.get_ids()], + [0.5812726, 0.5812725, 0.5812726, 0.5812726, 0.5813197, 0.6609682, + 0.5812726, 0.5812725, 0.5812726, 0.5813896]) + + if __name__ == '__main__': unittest.main() diff --git a/tests/fast_tests/test_examples.py b/tests/fast_tests/test_examples.py index b5faf6517..ea18aa415 100644 --- a/tests/fast_tests/test_examples.py +++ b/tests/fast_tests/test_examples.py @@ -69,8 +69,10 @@ def test_parse_args(self): 'aimsun': False, 'exp_config': 'exp_config', 'gen_emission': False, + 'is_baseline': False, 'no_render': False, - 'num_runs': 1 + 'num_runs': 1, + 'to_aws': None, }) # test the case when optional args are specified @@ -86,8 +88,10 @@ def test_parse_args(self): 'aimsun': True, 'exp_config': 'exp_config', 'gen_emission': True, + 'is_baseline': False, 'no_render': True, - 'num_runs': 2 + 'num_runs': 2, + 'to_aws': None, }) def test_bottleneck(self): @@ -148,9 +152,12 @@ def test_highway_single(self): @staticmethod def run_simulation(flow_params): + flow_params = deepcopy(flow_params) + # make the horizon small and set render to False flow_params['sim'].render = False flow_params['env'].horizon = 5 + flow_params['env'].warmup_steps = 0 # create an experiment object exp = Experiment(flow_params) @@ -167,12 +174,23 @@ def test_parse_args(self): args = parse_train_args(["exp_config"]) self.assertDictEqual(vars(args), { + 'algorithm': 'PPO', + 'checkpoint_freq': 20, 'exp_config': 'exp_config', + 'exp_title': None, + 'grid_search': False, + 'local_mode': False, 'rl_trainer': 'rllib', 'num_cpus': 1, + 'num_iterations': 200, + 'num_rollouts': 1, 'num_steps': 5000, + 'render': False, 'rollout_size': 1000, - 'checkpoint_path': None + 'checkpoint_path': None, + 'use_s3': False, + 'multi_node': False, + 'upload_graphs': None }) # test the case when optional args are specified @@ -183,15 +201,27 @@ def test_parse_args(self): "--num_steps", "3", "--rollout_size", "4", "--checkpoint_path", "5", + "--upload_graphs", "name", "strategy" ]) self.assertDictEqual(vars(args), { + 'algorithm': 'PPO', + 'checkpoint_freq': 20, 'checkpoint_path': '5', 'exp_config': 'exp_config', + 'exp_title': None, + 'grid_search': False, + 'local_mode': False, 'num_cpus': 1, + 'num_iterations': 200, + 'num_rollouts': 1, 'num_steps': 3, + 'render': False, 'rl_trainer': 'h-baselines', - 'rollout_size': 4 + 'rollout_size': 4, + 'use_s3': False, + 'multi_node': False, + 'upload_graphs': ['name', 'strategy'] }) @@ -203,6 +233,11 @@ class TestStableBaselineExamples(unittest.TestCase): """ @staticmethod def run_exp(flow_params): + # Reduce the number of warmup steps to speedup tests. + flow_params = deepcopy(flow_params) + flow_params['env'].warmup_steps = 0 + + # Run the example. train_model = run_stable_baselines_model(flow_params, 1, 4, 4) train_model.env.close() @@ -257,7 +292,7 @@ class TestRllibExamples(unittest.TestCase): """ def setUp(self): if not ray.is_initialized(): - ray.init(num_cpus=1) + ray.init(num_cpus=1, local_mode=True) def test_singleagent_figure_eight(self): self.run_exp(singleagent_figure_eight) @@ -357,7 +392,7 @@ def test_multiagent_i210(self): from examples.exp_configs.rl.multiagent.multiagent_i210 import POLICIES_TO_TRAIN as mi210pr from examples.exp_configs.rl.multiagent.multiagent_i210 import policy_mapping_fn as mi210mf - from ray.rllib.agents.ppo.ppo_policy import PPOTFPolicy + from ray.rllib.agents.ppo.ppo_tf_policy import PPOTFPolicy from ray.tune.registry import register_env from flow.utils.registry import make_create_env # test observation space 1 @@ -406,10 +441,16 @@ def test_multiagent_i210(self): @staticmethod def run_exp(flow_params, **kwargs): - alg_run, env_name, config = setup_rllib_exps(flow_params, 1, 1, **kwargs) + # Reduce the number of warmup steps to speedup tests. + flow_params = deepcopy(flow_params) + flow_params['env'].warmup_steps = 0 + + # Run the example. + alg_run, env_name, config = setup_rllib_exps( + flow_params, 1, 1, parse_train_args([""]), **kwargs) try: - ray.init(num_cpus=1) + ray.init(num_cpus=1, local_mode=True) except Exception as e: print("ERROR", e) config['train_batch_size'] = 50 @@ -436,7 +477,7 @@ def run_exp(flow_params, **kwargs): if __name__ == '__main__': try: - ray.init(num_cpus=1) + ray.init(num_cpus=1, local_mode=True) except Exception as e: print("ERROR", e) unittest.main() diff --git a/tests/fast_tests/test_experiment_base_class.py b/tests/fast_tests/test_experiment_base_class.py index 458af1027..8a7a9500c 100644 --- a/tests/fast_tests/test_experiment_base_class.py +++ b/tests/fast_tests/test_experiment_base_class.py @@ -189,6 +189,8 @@ def test_convert_to_csv(self): "speed", "headway", "leader_id", + "follower_id", + "leader_rel_speed", "target_accel_with_noise_with_failsafe", "target_accel_no_noise_no_failsafe", "target_accel_with_noise_no_failsafe", @@ -199,8 +201,6 @@ def test_convert_to_csv(self): "lane_number", "distance", "relative_position", - "follower_id", - "leader_rel_speed", ]) time.sleep(0.1) diff --git a/tests/fast_tests/test_files/energy_model_verification_data.csv b/tests/fast_tests/test_files/energy_model_verification_data.csv new file mode 100644 index 000000000..a3eb4ab0e --- /dev/null +++ b/tests/fast_tests/test_files/energy_model_verification_data.csv @@ -0,0 +1,78 @@ +rg(%),a(m/s^2),v(m/s),Tacoma(g/s),Tacoma_fit(g/s),Tacoma(gal/hr),Tacoma_fit(gal/hr),Tacoma(KW),Tacoma_fit(KW),Prius(SOC %/s),Prius_fit(KW) +0,-3,0,0.221939016,0.202898146,0.248342671,0.227036545,3.725140059,3.405548176,0,0.00104601 +0,-3,5,0.221939016,0,0.248342671,0,3.725140059,0,-3.55E-05,-11.69 +0,-3,10,0,0,0,0,0,0,-7.70E-05,-23.38 +0,-3,15,0,0,0,0,0,0,-0.000116458,-35.07 +0,-3,20,0,0,0,0,0,0,-0.000153287,-46.76 +0,-3,25,0,0,0,0,0,0,-0.000188081,-58.45 +0,-3,30,0,0,0,0,0,0,-0.000220255,-70.14 +0,-2.5,0,0.221939016,0.202898146,0.248342671,0.227036545,3.725140059,3.405548176,0,0.00104601 +0,-2.5,5,0.221939016,0,0.248342671,0,3.725140059,0,-3.55E-05,-11.69 +0,-2.5,10,0,0,0,0,0,0,-7.70E-05,-23.38 +0,-2.5,15,0,0,0,0,0,0,-0.000116458,-35.07 +0,-2.5,20,0,0,0,0,0,0,-0.000153287,-46.76 +0,-2.5,25,0,0,0,0,0,0,-0.000188081,-58.45 +0,-2.5,30,0,0,0,0,0,0,-0.000220255,-70.14 +0,-2,0,0.221939016,0.202898146,0.248342671,0.227036545,3.725140059,3.405548176,0,0.00104601 +0,-2,5,0.221939016,0,0.248342671,0,3.725140059,0,-3.55E-05,-11.69 +0,-2,10,0,0,0,0,0,0,-7.70E-05,-23.38 +0,-2,15,0,0,0,0,0,0,-0.000116458,-35.07 +0,-2,20,0,0,0,0,0,0,-0.000153287,-46.76 +0,-2,25,0,0,0,0,0,0,-0.000188081,-58.45 +0,-2,30,0,0,0,0,0,0,-0.000219899,-70.14 +0,-1.5,0,0.221939016,0.202898146,0.248342671,0.227036545,3.725140059,3.405548176,0,0.00104601 +0,-1.5,5,0.221939016,0,0.248342671,0,3.725140059,0,-3.26E-05,-10.27104516 +0,-1.5,10,0,0,0,0,0,0,-6.64E-05,-20.27904442 +0,-1.5,15,0,0,0,0,0,0,-9.76E-05,-29.77377487 +0,-1.5,20,0,0,0,0,0,0,-0.000125409,-38.50592258 +0,-1.5,25,0,0,0,0,0,0,-0.000149176,-46.22617363 +0,-1.5,30,0,0,0,0,0,0,-0.000168099,-52.68521408 +0,-1,0,0.221939016,0.202898146,0.248342671,0.227036545,3.725140059,3.405548176,0,0.00104601 +0,-1,5,0.221939016,0,0.248342671,0,3.725140059,0,-2.21E-05,-6.658177661 +0,-1,10,0,0,0,0,0,0,-4.41E-05,-13.05330942 +0,-1,15,0,0,0,0,0,0,-6.41E-05,-18.93517237 +0,-1,20,0,0,0,0,0,0,-8.11E-05,-24.05445258 +0,-1,25,0,0,0,0,0,0,-9.46E-05,-28.16183613 +0,-1,30,0,0,0,0,0,0,-0.000103742,-31.00800908 +0,-0.5,0,0.221939016,0.202898146,0.248342671,0.227036545,3.725140059,3.405548176,0,0.00104601 +0,-0.5,5,0.221939016,0,0.248342671,0,3.725140059,0,-1.05E-05,-3.045310161 +0,-0.5,10,0,0,0,0,0,0,-2.03E-05,-5.827574422 +0,-0.5,15,0,0,0,0,0,0,-2.83E-05,-8.096569872 +0,-0.5,20,0,0,0,0,0,0,-3.36E-05,-9.602982584 +0,-0.5,25,0,0,0,0,0,0,-3.55E-05,-10.09749863 +0,-0.5,30,0,0.022907406,0,0.025632656,0,0.384489841,-3.31E-05,-9.330804085 +0,0,0,0.200487388,0.202898146,0.224338984,0.227036545,3.365084763,3.405548176,0,0.00104601 +0,0,5,0.253341673,0.242980516,0.283481241,0.271887437,4.252218609,4.078311562,2.03E-06,0.653115465 +0,0,10,0.333366926,0.334679208,0.373026943,0.374495345,5.595404149,5.617430181,5.02E-06,1.60893047 +0,0,15,0.51256485,0.509457834,0.573543698,0.570067045,8.603155463,8.551005677,9.86E-06,3.155388525 +0,0,20,0.799694695,0.798780003,0.894832824,0.893809313,13.42249236,13.40713969,1.75E-05,5.57938713 +0,0,25,1.229256714,1.234109328,1.375499003,1.380928924,20.63248505,20.71393386,2.88E-05,9.167823785 +0,0,30,1.857240657,1.84690942,2.07819298,2.066632656,31.1728947,30.99948984,4.50E-05,14.20759599 +0,0.5,0,0.213329615,0.3398909,0.238709027,0.380327062,3.580635398,5.704905926,0,0.00104601 +0,0.5,5,0.645372234,0.829215641,0.722150919,0.927865819,10.83226378,13.91798729,1.57E-05,5.554765465 +0,0.5,10,0.962084653,1.370156703,1.076542,1.533161592,16.14813,22.99742388,3.20E-05,11.41223047 +0,0.5,15,1.492243064,1.994177699,1.669772331,2.231421157,25.04658496,33.47131735,5.06E-05,17.86033853 +0,0.5,20,2.171795319,2.732742239,2.4301696,3.057851289,36.45254401,45.86776934,7.25E-05,25.18598713 +0,0.5,25,2.994670552,3.617313935,3.350940707,4.047658766,50.2641106,60.71488149,9.90E-05,33.67607379 +0,0.5,30,4.337091333,4.679356397,4.853066687,5.236050363,72.79600031,78.54075544,0.000131784,43.61749599 +0,1,0,0.352945792,0.476883654,0.394935069,0.533617578,5.924026031,8.004263676,0,0.00104601 +0,1,5,1.068137902,1.415450765,1.195212199,1.583844201,17.92818298,23.75766301,3.04E-05,10.45641547 +0,1,10,1.719556415,2.405634198,1.924128711,2.691827839,28.86193066,40.37741758,6.08E-05,21.21553047 +0,1,15,2.579631321,3.478897565,2.886525062,3.892775268,43.29787592,58.39162903,9.43E-05,32.56528853 +0,1,20,3.592967151,4.666704475,4.020415493,5.221893266,60.30623239,78.32839899,0.00013214,44.79258713 +0,1,25,4.960534834,6.000518541,5.550680054,6.714388608,83.26020081,100.7158291,0.000176265,58.18432379 +0,1,30,8.468672195,7.511803374,9.47617372,8.405468069,142.1426058,126.082021,0.00022931,73.02739599 +0,1.5,0,0.546883865,0.613876408,0.611945579,0.686908095,9.179183688,10.30362143,0,0.00104601 +0,1.5,5,1.591785267,2.00168589,1.781156877,2.239822582,26.71735316,33.59733874,4.63E-05,15.35806547 +0,1.5,10,2.474883975,3.441111693,2.769316128,3.850494085,41.53974191,57.75741128,9.16E-05,31.01883047 +0,1.5,15,3.764036278,4.96361743,4.211836382,5.55412938,63.17754573,83.3119407,0.000141129,47.27023853 +0,1.5,20,5.510888585,6.600666711,6.166508324,7.385935243,92.49762487,110.7890286,0.000196989,64.39918713 +0,1.5,25,7.87103521,8.383723147,8.807437021,9.381118449,132.1115553,140.7167767,0.000261977,82.69257379 +0,1.5,30,11.00733306,10.34425035,12.31685416,11.57488578,184.7528123,173.6232866,0.000340665,102.437296 +0,2,0,0.771261491,0.750869162,0.863016977,0.840198612,12.94525465,12.60297918,0,0.00104601 +0,2,5,2.184066389,2.587921014,2.443900537,2.895800964,36.65850805,43.43701446,6.34E-05,20.25971547 +0,2,10,3.683640296,4.476589188,4.121875846,5.009160332,61.82813769,75.13740498,0.00012431,40.82213047 +0,2,15,6.181278245,6.448337296,6.916652943,7.215483492,103.7497941,108.2322524,0.000191549,61.97518853 +0,2,20,10.68072564,8.534628947,11.95139088,9.549977219,179.2708631,143.2496583,0.000268007,84.00578713 +0,2,25,18.85045788,10.76692775,21.09306034,12.04784829,316.3959051,180.7177244,0.000358314,107.2008238 +0,2,30,13.48024471,13.17669733,15.08396331,14.74430348,226.2594497,221.1645522,0.000470854,131.847196 diff --git a/tests/fast_tests/test_files/ring_230_emission.csv b/tests/fast_tests/test_files/ring_230_emission.csv index 9051074c8..342c5c7f3 100644 --- a/tests/fast_tests/test_files/ring_230_emission.csv +++ b/tests/fast_tests/test_files/ring_230_emission.csv @@ -1,117 +1,25 @@ -speed,CO,electricity,x,NOx,id,fuel,angle,time,edge_id,eclass,route,waiting,CO2,lane_number,PMx,type,noise,relative_position,HC,y -0.0,164.78,0.0,36.64,1.2,idm_0,1.13,94.02,0.1,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2624.72,0,0.07,idm,55.94,0.0,0.81,-1.65 -0.08,163.5,0.0,36.65,1.21,idm_0,1.13,94.01,0.2,bottom,HBEFA3/PC_G_EU4,routebottom,0.1,2631.03,0,0.07,idm,59.48,0.01,0.81,-1.65 -0.16,162.24,0.0,36.66,1.21,idm_0,1.13,93.98,0.3,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2636.67,0,0.07,idm,59.44,0.02,0.8,-1.65 -0.23,161.0,0.0,36.69,1.21,idm_0,1.14,93.94,0.4,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2641.63,0,0.07,idm,59.4,0.05,0.79,-1.65 -0.31,159.78,0.0,36.72,1.21,idm_0,1.14,93.88,0.5,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2645.91,0,0.06,idm,59.36,0.08,0.79,-1.65 -0.41,158.73,0.0,36.76,1.22,idm_0,1.15,93.8,0.6,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2679.14,0,0.07,idm,60.47,0.12,0.79,-1.65 -0.0,164.78,0.0,46.49,1.2,idm_1,1.13,78.81,0.1,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2624.72,0,0.07,idm,55.94,9.55,0.81,-0.34 -0.08,163.5,0.0,46.5,1.21,idm_1,1.13,78.8,0.2,bottom,HBEFA3/PC_G_EU4,routebottom,0.1,2631.03,0,0.07,idm,59.48,9.55,0.81,-0.33 -0.16,162.24,0.0,46.51,1.21,idm_1,1.13,78.78,0.3,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2636.67,0,0.07,idm,59.44,9.57,0.8,-0.33 -0.23,161.0,0.0,46.54,1.21,idm_1,1.14,78.74,0.4,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2641.63,0,0.07,idm,59.4,9.59,0.79,-0.32 -0.31,159.78,0.0,46.57,1.21,idm_1,1.14,78.7,0.5,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2645.91,0,0.06,idm,59.36,9.62,0.79,-0.31 -0.41,158.73,0.0,46.61,1.22,idm_1,1.15,78.64,0.6,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2679.14,0,0.07,idm,60.47,9.66,0.79,-0.3 -0.0,164.78,0.0,56.08,1.2,idm_10,1.13,304.55,0.1,right,HBEFA3/PC_G_EU4,routeright,0.0,2624.72,0,0.07,idm,55.94,37.95,0.81,69.53 -0.08,163.5,0.0,56.08,1.21,idm_10,1.13,304.54,0.2,right,HBEFA3/PC_G_EU4,routeright,0.1,2631.03,0,0.07,idm,59.48,37.96,0.81,69.53 -0.16,162.24,0.0,56.06,1.21,idm_10,1.13,304.52,0.3,right,HBEFA3/PC_G_EU4,routeright,0.0,2636.67,0,0.07,idm,59.44,37.98,0.8,69.54 -0.23,161.0,0.0,56.04,1.21,idm_10,1.14,304.48,0.4,right,HBEFA3/PC_G_EU4,routeright,0.0,2641.63,0,0.07,idm,59.4,38.0,0.79,69.55 -0.31,159.78,0.0,56.01,1.21,idm_10,1.14,304.44,0.5,right,HBEFA3/PC_G_EU4,routeright,0.0,2645.91,0,0.06,idm,59.36,38.03,0.79,69.57 -0.41,158.73,0.0,55.98,1.22,idm_10,1.15,304.38,0.6,right,HBEFA3/PC_G_EU4,routeright,0.0,2679.14,0,0.07,idm,60.47,38.07,0.79,69.59 -0.0,164.78,0.0,46.95,1.2,idm_11,1.13,289.47,0.1,right,HBEFA3/PC_G_EU4,routeright,0.0,2624.72,0,0.07,idm,55.94,47.5,0.81,73.43 -0.08,163.5,0.0,46.94,1.21,idm_11,1.13,289.45,0.2,right,HBEFA3/PC_G_EU4,routeright,0.1,2631.03,0,0.07,idm,59.48,47.51,0.81,73.43 -0.16,162.24,0.0,46.92,1.21,idm_11,1.13,289.42,0.3,right,HBEFA3/PC_G_EU4,routeright,0.0,2636.67,0,0.07,idm,59.44,47.52,0.8,73.44 -0.23,161.0,0.0,46.9,1.21,idm_11,1.14,289.38,0.4,right,HBEFA3/PC_G_EU4,routeright,0.0,2641.63,0,0.07,idm,59.4,47.55,0.79,73.44 -0.31,159.78,0.0,46.87,1.21,idm_11,1.14,289.32,0.5,right,HBEFA3/PC_G_EU4,routeright,0.0,2645.91,0,0.06,idm,59.36,47.58,0.79,73.45 -0.41,158.73,0.0,46.83,1.22,idm_11,1.15,289.24,0.6,right,HBEFA3/PC_G_EU4,routeright,0.0,2679.14,0,0.07,idm,60.47,47.62,0.79,73.46 -0.0,164.78,0.0,37.11,1.2,idm_12,1.13,274.71,0.1,right,HBEFA3/PC_G_EU4,routeright,0.0,2624.72,0,0.07,idm,55.94,57.05,0.81,74.86 -0.08,163.5,0.0,37.11,1.21,idm_12,1.13,274.7,0.2,right,HBEFA3/PC_G_EU4,routeright,0.1,2631.03,0,0.07,idm,59.48,57.05,0.81,74.86 -0.16,162.24,0.0,37.09,1.21,idm_12,1.13,274.68,0.3,right,HBEFA3/PC_G_EU4,routeright,0.0,2636.67,0,0.07,idm,59.44,57.07,0.8,74.86 -0.23,161.0,0.0,37.07,1.21,idm_12,1.14,274.65,0.4,right,HBEFA3/PC_G_EU4,routeright,0.0,2641.63,0,0.07,idm,59.4,57.09,0.79,74.86 -0.31,159.78,0.0,37.03,1.21,idm_12,1.14,274.6,0.5,right,HBEFA3/PC_G_EU4,routeright,0.0,2645.91,0,0.06,idm,59.36,57.12,0.79,74.86 -0.41,158.73,0.0,36.99,1.22,idm_12,1.15,274.55,0.6,right,HBEFA3/PC_G_EU4,routeright,0.0,2679.14,0,0.07,idm,60.47,57.16,0.79,74.86 -0.0,164.78,0.0,27.19,1.2,idm_13,1.13,259.6,0.1,top,HBEFA3/PC_G_EU4,routetop,0.0,2624.72,0,0.07,idm,55.94,9.09,0.81,73.68 -0.08,163.5,0.0,27.18,1.21,idm_13,1.13,259.58,0.2,top,HBEFA3/PC_G_EU4,routetop,0.1,2631.03,0,0.07,idm,59.48,9.1,0.81,73.68 -0.16,162.24,0.0,27.17,1.21,idm_13,1.13,259.55,0.3,top,HBEFA3/PC_G_EU4,routetop,0.0,2636.67,0,0.07,idm,59.44,9.11,0.8,73.67 -0.23,161.0,0.0,27.14,1.21,idm_13,1.14,259.51,0.4,top,HBEFA3/PC_G_EU4,routetop,0.0,2641.63,0,0.07,idm,59.4,9.14,0.79,73.67 -0.31,159.78,0.0,27.11,1.21,idm_13,1.14,259.45,0.5,top,HBEFA3/PC_G_EU4,routetop,0.0,2645.91,0,0.06,idm,59.36,9.17,0.79,73.66 -0.41,158.73,0.0,27.07,1.22,idm_13,1.15,259.37,0.6,top,HBEFA3/PC_G_EU4,routetop,0.0,2679.14,0,0.07,idm,60.47,9.21,0.79,73.65 -0.0,164.78,0.0,17.96,1.2,idm_14,1.13,244.67,0.1,top,HBEFA3/PC_G_EU4,routetop,0.0,2624.72,0,0.07,idm,55.94,18.64,0.81,70.0 -0.08,163.5,0.0,17.95,1.21,idm_14,1.13,244.66,0.2,top,HBEFA3/PC_G_EU4,routetop,0.1,2631.03,0,0.07,idm,59.48,18.64,0.81,70.0 -0.16,162.24,0.0,17.94,1.21,idm_14,1.13,244.63,0.3,top,HBEFA3/PC_G_EU4,routetop,0.0,2636.67,0,0.07,idm,59.44,18.66,0.8,69.99 -0.23,161.0,0.0,17.92,1.21,idm_14,1.14,244.6,0.4,top,HBEFA3/PC_G_EU4,routetop,0.0,2641.63,0,0.07,idm,59.4,18.68,0.79,69.98 -0.31,159.78,0.0,17.89,1.21,idm_14,1.14,244.55,0.5,top,HBEFA3/PC_G_EU4,routetop,0.0,2645.91,0,0.06,idm,59.36,18.71,0.79,69.96 -0.0,164.78,0.0,9.98,1.2,idm_15,1.13,229.84,0.1,top,HBEFA3/PC_G_EU4,routetop,0.0,2624.72,0,0.07,idm,55.94,28.18,0.81,64.07 -0.08,163.5,0.0,9.98,1.21,idm_15,1.13,229.83,0.2,top,HBEFA3/PC_G_EU4,routetop,0.1,2631.03,0,0.07,idm,59.48,28.19,0.81,64.07 -0.16,162.24,0.0,9.97,1.21,idm_15,1.13,229.8,0.3,top,HBEFA3/PC_G_EU4,routetop,0.0,2636.67,0,0.07,idm,59.44,28.21,0.8,64.06 -0.23,161.0,0.0,9.95,1.21,idm_15,1.14,229.76,0.4,top,HBEFA3/PC_G_EU4,routetop,0.0,2641.63,0,0.07,idm,59.4,28.23,0.79,64.04 -0.31,159.78,0.0,9.93,1.21,idm_15,1.14,229.7,0.5,top,HBEFA3/PC_G_EU4,routetop,0.0,2645.91,0,0.06,idm,59.36,28.26,0.79,64.02 -0.0,164.78,0.0,3.81,1.2,idm_16,1.13,214.88,0.1,top,HBEFA3/PC_G_EU4,routetop,0.0,2624.72,0,0.07,idm,55.94,37.73,0.81,56.29 -0.08,163.5,0.0,3.81,1.21,idm_16,1.13,214.87,0.2,top,HBEFA3/PC_G_EU4,routetop,0.1,2631.03,0,0.07,idm,59.48,37.74,0.81,56.28 -0.16,162.24,0.0,3.8,1.21,idm_16,1.13,214.85,0.3,top,HBEFA3/PC_G_EU4,routetop,0.0,2636.67,0,0.07,idm,59.44,37.75,0.8,56.27 -0.23,161.0,0.0,3.79,1.21,idm_16,1.14,214.81,0.4,top,HBEFA3/PC_G_EU4,routetop,0.0,2641.63,0,0.07,idm,59.4,37.77,0.79,56.24 -0.31,159.78,0.0,3.77,1.21,idm_16,1.14,214.77,0.5,top,HBEFA3/PC_G_EU4,routetop,0.0,2645.91,0,0.06,idm,59.36,37.81,0.79,56.22 -0.0,164.78,0.0,-0.15,1.2,idm_17,1.13,199.9,0.1,top,HBEFA3/PC_G_EU4,routetop,0.0,2624.72,0,0.07,idm,55.94,47.27,0.81,47.18 -0.08,163.5,0.0,-0.15,1.21,idm_17,1.13,199.88,0.2,top,HBEFA3/PC_G_EU4,routetop,0.1,2631.03,0,0.07,idm,59.48,47.28,0.81,47.17 -0.16,162.24,0.0,-0.16,1.21,idm_17,1.13,199.85,0.3,top,HBEFA3/PC_G_EU4,routetop,0.0,2636.67,0,0.07,idm,59.44,47.3,0.8,47.15 -0.23,161.0,0.0,-0.16,1.21,idm_17,1.14,199.81,0.4,top,HBEFA3/PC_G_EU4,routetop,0.0,2641.63,0,0.07,idm,59.4,47.32,0.79,47.13 -0.31,159.78,0.0,-0.17,1.21,idm_17,1.14,199.75,0.5,top,HBEFA3/PC_G_EU4,routetop,0.0,2645.91,0,0.06,idm,59.36,47.35,0.79,47.1 -0.0,164.78,0.0,-1.64,1.2,idm_18,1.13,185.04,0.1,top,HBEFA3/PC_G_EU4,routetop,0.0,2624.72,0,0.07,idm,55.94,56.82,0.81,37.35 -0.08,163.5,0.0,-1.64,1.21,idm_18,1.13,185.03,0.2,top,HBEFA3/PC_G_EU4,routetop,0.1,2631.03,0,0.07,idm,59.48,56.83,0.81,37.34 -0.16,162.24,0.0,-1.64,1.21,idm_18,1.13,185.0,0.3,top,HBEFA3/PC_G_EU4,routetop,0.0,2636.67,0,0.07,idm,59.44,56.84,0.8,37.33 -0.23,161.0,0.0,-1.64,1.21,idm_18,1.14,184.97,0.4,top,HBEFA3/PC_G_EU4,routetop,0.0,2641.63,0,0.07,idm,59.4,56.87,0.79,37.3 -0.31,159.78,0.0,-1.64,1.21,idm_18,1.14,184.93,0.5,top,HBEFA3/PC_G_EU4,routetop,0.0,2645.91,0,0.06,idm,59.36,56.9,0.79,37.27 -0.0,164.78,0.0,-0.52,1.2,idm_19,1.13,170.03,0.1,left,HBEFA3/PC_G_EU4,routeleft,0.0,2624.72,0,0.07,idm,55.94,8.86,0.81,27.42 -0.08,163.5,0.0,-0.52,1.21,idm_19,1.13,170.01,0.2,left,HBEFA3/PC_G_EU4,routeleft,0.1,2631.03,0,0.07,idm,59.48,8.87,0.81,27.41 -0.16,162.24,0.0,-0.51,1.21,idm_19,1.13,169.98,0.3,left,HBEFA3/PC_G_EU4,routeleft,0.0,2636.67,0,0.07,idm,59.44,8.89,0.8,27.39 -0.23,161.0,0.0,-0.51,1.21,idm_19,1.14,169.94,0.4,left,HBEFA3/PC_G_EU4,routeleft,0.0,2641.63,0,0.07,idm,59.4,8.91,0.79,27.37 -0.31,159.78,0.0,-0.5,1.21,idm_19,1.14,169.88,0.5,left,HBEFA3/PC_G_EU4,routeleft,0.0,2645.91,0,0.06,idm,59.36,8.94,0.79,27.34 -0.0,164.78,0.0,55.68,1.2,idm_2,1.13,64.0,0.1,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2624.72,0,0.07,idm,55.94,19.09,0.81,3.45 -0.08,163.5,0.0,55.68,1.21,idm_2,1.13,63.99,0.2,bottom,HBEFA3/PC_G_EU4,routebottom,0.1,2631.03,0,0.07,idm,59.48,19.1,0.81,3.45 -0.16,162.24,0.0,55.7,1.21,idm_2,1.13,63.97,0.3,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2636.67,0,0.07,idm,59.44,19.11,0.8,3.46 -0.23,161.0,0.0,55.72,1.21,idm_2,1.14,63.93,0.4,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2641.63,0,0.07,idm,59.4,19.14,0.79,3.47 -0.31,159.78,0.0,55.75,1.21,idm_2,1.14,63.88,0.5,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2645.91,0,0.06,idm,59.36,19.17,0.79,3.49 -0.0,164.78,0.0,3.11,1.2,idm_20,1.13,155.0,0.1,left,HBEFA3/PC_G_EU4,routeleft,0.0,2624.72,0,0.07,idm,55.94,18.41,0.81,18.17 -0.08,163.5,0.0,3.11,1.21,idm_20,1.13,154.99,0.2,left,HBEFA3/PC_G_EU4,routeleft,0.1,2631.03,0,0.07,idm,59.48,18.42,0.81,18.16 -0.16,162.24,0.0,3.12,1.21,idm_20,1.13,154.96,0.3,left,HBEFA3/PC_G_EU4,routeleft,0.0,2636.68,0,0.07,idm,59.44,18.43,0.8,18.15 -0.23,161.0,0.0,3.13,1.21,idm_20,1.14,154.93,0.4,left,HBEFA3/PC_G_EU4,routeleft,0.0,2641.7,0,0.07,idm,59.41,18.46,0.79,18.12 -0.31,159.77,0.0,3.15,1.21,idm_20,1.14,154.89,0.5,left,HBEFA3/PC_G_EU4,routeleft,0.0,2646.14,0,0.06,idm,59.37,18.49,0.79,18.1 -0.0,164.78,0.0,8.98,1.2,idm_21,1.13,140.22,0.1,left,HBEFA3/PC_G_EU4,routeleft,0.0,2624.72,0,0.07,idm,55.94,27.95,0.81,10.15 -0.1,163.3,0.0,8.99,1.21,idm_21,1.13,140.21,0.2,left,HBEFA3/PC_G_EU4,routeleft,0.1,2637.25,0,0.07,idm,60.3,27.96,0.81,10.15 -0.2,161.84,0.0,9.0,1.21,idm_21,1.14,140.18,0.3,left,HBEFA3/PC_G_EU4,routeleft,0.0,2649.89,0,0.07,idm,60.34,27.98,0.8,10.13 -0.29,160.38,0.0,9.02,1.21,idm_21,1.14,140.14,0.4,left,HBEFA3/PC_G_EU4,routeleft,0.0,2662.63,0,0.07,idm,60.37,28.01,0.79,10.11 -0.39,158.94,0.0,9.05,1.22,idm_21,1.15,140.07,0.5,left,HBEFA3/PC_G_EU4,routeleft,0.0,2675.48,0,0.07,idm,60.41,28.05,0.79,10.08 -0.0,164.78,0.0,63.57,1.2,idm_3,1.13,49.05,0.1,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2624.72,0,0.07,idm,55.94,28.64,0.81,9.48 -0.08,163.5,0.0,63.58,1.21,idm_3,1.13,49.04,0.2,bottom,HBEFA3/PC_G_EU4,routebottom,0.1,2631.03,0,0.07,idm,59.48,28.64,0.81,9.49 -0.16,162.24,0.0,63.59,1.21,idm_3,1.13,49.02,0.3,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2636.67,0,0.07,idm,59.44,28.66,0.8,9.5 -0.23,161.0,0.0,63.61,1.21,idm_3,1.14,48.99,0.4,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2641.63,0,0.07,idm,59.4,28.68,0.79,9.52 -0.31,159.78,0.0,63.63,1.21,idm_3,1.14,48.94,0.5,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2645.91,0,0.06,idm,59.36,28.71,0.79,9.54 -0.0,164.78,0.0,69.65,1.2,idm_4,1.13,34.22,0.1,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2624.72,0,0.07,idm,55.94,38.18,0.81,17.34 -0.08,163.5,0.0,69.65,1.21,idm_4,1.13,34.21,0.2,bottom,HBEFA3/PC_G_EU4,routebottom,0.1,2631.03,0,0.07,idm,59.48,38.19,0.81,17.35 -0.16,162.24,0.0,69.66,1.21,idm_4,1.13,34.19,0.3,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2636.67,0,0.07,idm,59.44,38.21,0.8,17.36 -0.23,161.0,0.0,69.68,1.21,idm_4,1.14,34.15,0.4,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2641.63,0,0.07,idm,59.4,38.23,0.79,17.38 -0.31,159.78,0.0,69.69,1.21,idm_4,1.14,34.11,0.5,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2645.91,0,0.06,idm,59.36,38.26,0.79,17.41 -0.0,164.78,0.0,73.49,1.2,idm_5,1.13,19.04,0.1,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2624.72,0,0.07,idm,55.94,47.73,0.81,26.5 -0.08,163.5,0.0,73.5,1.21,idm_5,1.13,19.02,0.2,bottom,HBEFA3/PC_G_EU4,routebottom,0.1,2631.03,0,0.07,idm,59.48,47.74,0.81,26.51 -0.16,162.24,0.0,73.5,1.21,idm_5,1.13,18.99,0.3,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2636.67,0,0.07,idm,59.44,47.75,0.8,26.53 -0.23,161.0,0.0,73.51,1.21,idm_5,1.14,18.95,0.4,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2641.63,0,0.07,idm,59.4,47.77,0.79,26.55 -0.31,159.78,0.0,73.52,1.21,idm_5,1.14,18.91,0.5,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2645.91,0,0.06,idm,59.36,47.81,0.79,26.58 -0.0,164.78,0.0,74.87,1.2,idm_6,1.13,4.39,0.1,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2624.72,0,0.07,idm,55.94,57.27,0.81,36.34 -0.08,163.5,0.0,74.87,1.21,idm_6,1.13,4.38,0.2,bottom,HBEFA3/PC_G_EU4,routebottom,0.1,2631.03,0,0.07,idm,59.48,57.28,0.81,36.35 -0.16,162.24,0.0,74.87,1.21,idm_6,1.13,4.36,0.3,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2636.67,0,0.07,idm,59.44,57.3,0.8,36.37 -0.23,161.0,0.0,74.87,1.21,idm_6,1.14,4.32,0.4,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2641.63,0,0.07,idm,59.4,57.32,0.79,36.39 -0.31,159.78,0.0,74.87,1.21,idm_6,1.14,4.28,0.5,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2645.91,0,0.06,idm,59.36,57.35,0.79,36.42 -0.0,164.78,0.0,73.62,1.2,idm_7,1.13,349.16,0.1,right,HBEFA3/PC_G_EU4,routeright,0.0,2624.72,0,0.07,idm,55.94,9.32,0.81,46.26 -0.08,163.5,0.0,73.62,1.21,idm_7,1.13,349.15,0.2,right,HBEFA3/PC_G_EU4,routeright,0.1,2631.03,0,0.07,idm,59.48,9.33,0.81,46.27 -0.16,162.24,0.0,73.61,1.21,idm_7,1.13,349.12,0.3,right,HBEFA3/PC_G_EU4,routeright,0.0,2636.67,0,0.07,idm,59.44,9.34,0.8,46.28 -0.23,161.0,0.0,73.6,1.21,idm_7,1.14,349.07,0.4,right,HBEFA3/PC_G_EU4,routeright,0.0,2641.63,0,0.07,idm,59.4,9.37,0.79,46.31 -0.31,159.78,0.0,73.6,1.21,idm_7,1.14,349.01,0.5,right,HBEFA3/PC_G_EU4,routeright,0.0,2645.91,0,0.06,idm,59.36,9.4,0.79,46.34 -0.0,164.78,0.0,69.89,1.2,idm_8,1.13,334.33,0.1,right,HBEFA3/PC_G_EU4,routeright,0.0,2624.72,0,0.07,idm,55.94,18.86,0.81,55.47 -0.08,163.5,0.0,69.88,1.21,idm_8,1.13,334.32,0.2,right,HBEFA3/PC_G_EU4,routeright,0.1,2631.03,0,0.07,idm,59.48,18.87,0.81,55.47 -0.16,162.24,0.0,69.87,1.21,idm_8,1.13,334.3,0.3,right,HBEFA3/PC_G_EU4,routeright,0.0,2636.67,0,0.07,idm,59.44,18.89,0.8,55.49 -0.23,161.0,0.0,69.86,1.21,idm_8,1.14,334.27,0.4,right,HBEFA3/PC_G_EU4,routeright,0.0,2641.63,0,0.07,idm,59.4,18.91,0.79,55.51 -0.31,159.78,0.0,69.85,1.21,idm_8,1.14,334.22,0.5,right,HBEFA3/PC_G_EU4,routeright,0.0,2645.91,0,0.06,idm,59.36,18.94,0.79,55.54 -0.0,164.78,0.0,63.91,1.2,idm_9,1.13,319.44,0.1,right,HBEFA3/PC_G_EU4,routeright,0.0,2624.72,0,0.07,idm,55.94,28.41,0.81,63.4 -0.08,163.5,0.0,63.9,1.21,idm_9,1.13,319.42,0.2,right,HBEFA3/PC_G_EU4,routeright,0.1,2631.03,0,0.07,idm,59.48,28.42,0.81,63.41 -0.16,162.24,0.0,63.89,1.21,idm_9,1.13,319.39,0.3,right,HBEFA3/PC_G_EU4,routeright,0.0,2636.67,0,0.07,idm,59.44,28.43,0.8,63.42 -0.23,161.0,0.0,63.87,1.21,idm_9,1.14,319.35,0.4,right,HBEFA3/PC_G_EU4,routeright,0.0,2641.63,0,0.07,idm,59.4,28.46,0.79,63.44 -0.31,159.78,0.0,63.85,1.21,idm_9,1.14,319.3,0.5,right,HBEFA3/PC_G_EU4,routeright,0.0,2645.91,0,0.06,idm,59.36,28.49,0.79,63.46 +time,id,x,y,speed,headway,leader_id,follower_id,leader_rel_speed,target_accel_with_noise_with_failsafe,target_accel_no_noise_no_failsafe,target_accel_with_noise_no_failsafe,target_accel_no_noise_with_failsafe,realized_accel,road_grade,edge_id,lane_number,distance,relative_position +0.0,idm_0,36.64,-1.6,0.0,4.545454545454547,idm_1,idm_21,0.0,0.0,0.0,0.0,0.0,0.0,0,bottom,0,0.0,0.0 +0.1,idm_0,36.648322761506634,-1.599834647122385,0.07984158415841586,4.545454545454546,idm_1,idm_21,0.0,0.8064000000000001,0.8064000000000001,0.8064000000000001,0.8064000000000001,0.7984158415841586,0,bottom,0,0.007984158415841587,0.007984158415841587 +0.2,idm_0,36.66480556684144,-1.599507174168713,0.15812219156578355,4.545454545454545,idm_1,idm_21,0.0,0.7906341348144134,0.7906341348144134,0.7906341348144134,0.7906341348144134,0.7828060740736771,0,bottom,0,0.023796377572419945,0.023796377572419945 +0.3,idm_0,36.68928269645688,-1.599020873580327,0.23481302481051264,4.545454545454546,idm_1,idm_21,5.551115123125783e-17,0.7745774157717638,0.7745774157717638,0.7745774157717638,0.7745774157717638,0.7669083324472908,0,bottom,0,0.04727768005347121,0.04727768005347121 +0.0,idm_1,46.477059895666216,-0.2910450274933619,0.0,4.545454545454547,idm_2,idm_0,0.0,0.0,0.0,0.0,0.0,0.0,0,bottom,0,0.0,9.545454545454547 +0.1,idm_1,46.48510950976829,-0.2889238453988948,0.07984158415841586,4.545454545454547,idm_2,idm_0,0.0,0.8064000000000001,0.8064000000000001,0.8064000000000001,0.8064000000000001,0.7984158415841586,0,bottom,0,0.007984158415840879,9.553438703870388 +0.2,idm_1,46.5010513605782,-0.2847229522800698,0.15812219156578355,4.5454545454545485,idm_2,idm_0,0.0,0.7906341348144135,0.7906341348144135,0.7906341348144135,0.7906341348144135,0.7828060740736771,0,bottom,0,0.023796377572418592,9.569250923026964 +0.3,idm_1,46.524725167351825,-0.2784845842789108,0.2348130248105127,4.5454545454545485,idm_2,idm_0,-5.551115123125783e-17,0.7745774157717642,0.7745774157717642,0.7745774157717642,0.7745774157717642,0.7669083324472914,0,bottom,0,0.04727768005347066,9.592732225508016 +0.0,idm_2,55.65270828548022,3.488595652781747,0.0,4.545454545454547,idm_3,idm_1,0.0,0.0,0.0,0.0,0.0,0.0,0,bottom,0,0.0,19.090909090909093 +0.1,idm_2,55.66000796138611,3.4925969566116453,0.07984158415841586,4.545454545454547,idm_3,idm_1,0.0,0.8064000000000001,0.8064000000000001,0.8064000000000001,0.8064000000000001,0.7984158415841586,0,bottom,0,0.007984158415840879,19.098893249324934 +0.2,idm_2,55.67446459778839,3.5005213350840068,0.15812219156578355,4.545454545454547,idm_3,idm_1,0.0,0.7906341348144135,0.7906341348144135,0.7906341348144135,0.7906341348144135,0.7828060740736771,0,bottom,0,0.02379637757242037,19.114705468481514 +0.3,idm_2,55.69593284641682,3.5122891158136613,0.23481302481051264,4.545454545454547,idm_3,idm_1,5.551115123125783e-17,0.7745774157717641,0.7745774157717641,0.7745774157717641,0.7745774157717641,0.7669083324472908,0,bottom,0,0.04727768005347244,19.138186770962566 +0.0,idm_3,63.54122270574333,9.511222705743334,0.0,4.545454545454547,idm_4,idm_2,0.0,0.0,0.0,0.0,0.0,0.0,0,bottom,0,0.0,28.63636363636364 +0.1,idm_3,63.54710894820649,9.517108948206497,0.07984158415841586,4.545454545454549,idm_4,idm_2,0.0,0.8064000000000001,0.8064000000000001,0.8064000000000001,0.8064000000000001,0.7984158415841586,0,bottom,0,0.007984158415840879,28.64434779477948 +0.2,idm_3,63.558766351653254,9.528766351653257,0.15812219156578355,4.545454545454549,idm_4,idm_2,0.0,0.7906341348144138,0.7906341348144138,0.7906341348144138,0.7906341348144138,0.7828060740736771,0,bottom,0,0.02379637757242037,28.660160013936057 +0.3,idm_3,63.57607771154312,9.546077711543122,0.2348130248105127,4.545454545454549,idm_4,idm_2,-5.551115123125783e-17,0.7745774157717643,0.7745774157717643,0.7745774157717643,0.7745774157717643,0.7669083324472914,0,bottom,0,0.04727768005347244,28.683641316417113 +0.0,idm_4,69.61055207686752,17.363529025870548,0.0,4.545454545454547,idm_5,idm_3,0.0,0.0,0.0,0.0,0.0,0.0,0,bottom,0,0.0,38.18181818181819 +0.1,idm_4,69.61489064350748,17.370633428743492,0.07984158415841586,4.545454545454547,idm_5,idm_3,0.0,0.8064000000000001,0.8064000000000001,0.8064000000000001,0.8064000000000001,0.7984158415841586,0,bottom,0,0.007984158415844433,38.18980234023403 +0.2,idm_4,69.62348295380036,17.384703336848084,0.15812219156578355,4.545454545454547,idm_5,idm_3,0.0,0.7906341348144135,0.7906341348144135,0.7906341348144135,0.7906341348144135,0.7828060740736771,0,bottom,0,0.023796377572423918,38.20561455939061 +0.3,idm_4,69.63624261991681,17.40559729011379,0.23481302481051264,4.545454545454547,idm_5,idm_3,0.0,0.7745774157717641,0.7745774157717641,0.7745774157717641,0.7745774157717641,0.7669083324472908,0,bottom,0,0.04727768005347599,38.229095861871656 +0.0,idm_5,73.45066460734851,26.51380415096358,0.0,4.545454545454547,idm_6,idm_4,0.0,0.0,0.0,0.0,0.0,0.0,0,bottom,0,0.0,47.72727272727274 +0.1,idm_5,73.45278578944298,26.521853765065657,0.07984158415841586,4.545454545454547,idm_6,idm_4,-1.3877787807814454e-17,0.8064000000000001,0.8064000000000001,0.8064000000000001,0.8064000000000001,0.7984158415841586,0,bottom,0,0.007984158415844433,47.73525688568858 +0.2,idm_5,73.4569866825618,26.53779561587557,0.15812219156578355,4.545454545454547,idm_6,idm_4,-5.551115123125783e-17,0.7906341348144135,0.7906341348144135,0.7906341348144135,0.7906341348144135,0.7828060740736771,0,bottom,0,0.023796377572423918,47.75106910484515 +0.3,idm_5,73.46322505056297,26.561469422649196,0.23481302481051264,4.545454545454547,idm_6,idm_4,-1.1102230246251563e-16,0.7745774157717641,0.7745774157717641,0.7745774157717641,0.7745774157717641,0.7669083324472908,0,bottom,0,0.04727768005347599,47.77455040732621 diff --git a/tests/fast_tests/test_files/test_save_state.xml b/tests/fast_tests/test_files/test_save_state.xml new file mode 100644 index 000000000..13d4a84b3 --- /dev/null +++ b/tests/fast_tests/test_files/test_save_state.xml @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/fast_tests/test_replays.py b/tests/fast_tests/test_replays.py new file mode 100644 index 000000000..bc151d2d1 --- /dev/null +++ b/tests/fast_tests/test_replays.py @@ -0,0 +1,53 @@ +from flow.replay import rllib_replay as rl_replay +from flow.replay.rllib_replay import replay_rllib + +import os +import unittest +import ray + +os.environ['TEST_FLAG'] = 'True' + + +class TestRLReplay(unittest.TestCase): + """Tests rllib_replay""" + + def test_rllib_replay_single(self): + """Test for single agent replay""" + try: + ray.init(num_cpus=1) + except Exception: + pass + # current path + current_path = os.path.realpath(__file__).rsplit('/', 1)[0] + + # run the experiment and check it doesn't crash + arg_str = '{}/../data/rllib_data/single_agent 1 --num_rollouts 1 ' \ + '--render_mode no_render ' \ + '--horizon 10'.format(current_path).split() + parser = rl_replay.create_parser() + pass_args = parser.parse_args(arg_str) + replay_rllib(pass_args) + + # FIXME(ev) set the horizon so that this runs faster + def test_rllib_replay_multi(self): + """Test for multi-agent replay""" + try: + ray.init(num_cpus=1) + except Exception: + pass + # current path + current_path = os.path.realpath(__file__).rsplit('/', 1)[0] + + # run the experiment and check it doesn't crash + arg_str = '{}/../data/rllib_data/multi_agent 1 --num_rollouts 1 ' \ + '--render_mode no_render ' \ + '--horizon 10'.format(current_path).split() + parser = rl_replay.create_parser() + pass_args = parser.parse_args(arg_str) + replay_rllib(pass_args) + + +if __name__ == '__main__': + ray.init(num_cpus=1) + unittest.main() + ray.shutdown() diff --git a/tests/fast_tests/test_rewards.py b/tests/fast_tests/test_rewards.py index 3f2e08cde..ac406b545 100644 --- a/tests/fast_tests/test_rewards.py +++ b/tests/fast_tests/test_rewards.py @@ -7,7 +7,6 @@ from flow.core.rewards import average_velocity, min_delay from flow.core.rewards import desired_velocity, boolean_action_penalty from flow.core.rewards import penalize_near_standstill, penalize_standstill -from flow.core.rewards import energy_consumption os.environ["TEST_FLAG"] = "True" @@ -152,31 +151,6 @@ def test_penalize_near_standstill(self): self.assertEqual(penalize_near_standstill(env, thresh=2), -10) self.assertEqual(penalize_near_standstill(env, thresh=0.5), -9) - def test_energy_consumption(self): - """Test the energy consumption method.""" - vehicles = VehicleParams() - vehicles.add("test", num_vehicles=10) - - env_params = EnvParams(additional_params={ - "target_velocity": 10, "max_accel": 1, "max_decel": 1, - "sort_vehicles": False}) - - env, _, _ = ring_road_exp_setup(vehicles=vehicles, - env_params=env_params) - - # check the penalty is zero at speed zero - self.assertEqual(energy_consumption(env, gain=1), 0) - - # change the speed of one vehicle - env.k.vehicle.test_set_speed("test_0", 1) - self.assertEqual(energy_consumption(env), -12.059337750000001) - - # check that stepping change the previous speeds and increases the energy consumption - env.step(rl_actions=None) - env.step(rl_actions=None) - self.assertGreater(env.k.vehicle.get_previous_speed("test_0"), 0.0) - self.assertLess(energy_consumption(env), -12.059337750000001) - def test_boolean_action_penalty(self): """Test the boolean_action_penalty method.""" actions = [False, False, False, False, False] diff --git a/tests/fast_tests/test_scenario_base_class.py b/tests/fast_tests/test_scenario_base_class.py index e563c07ab..8f90fa386 100644 --- a/tests/fast_tests/test_scenario_base_class.py +++ b/tests/fast_tests/test_scenario_base_class.py @@ -1057,5 +1057,33 @@ def test_network_template(self): self.assertDictEqual(network.routes, expected_routes) +class TestMaxSpeedAcquisiton(unittest.TestCase): + """Tests getting and setting max speed on an edge""" + + def test_num_lanes_junctions(self): + """ + Tests the num_lanes() method when called on junctions + """ + additional_net_params = { + "radius_ring": 30, + "lanes": 3, + "speed_limit": 60, + "resolution": 40 + } + net_params = NetParams(additional_params=additional_net_params) + + env, network, _ = figure_eight_exp_setup(net_params=net_params) + + self.assertEqual(env.k.network.get_max_speed("bottom", 0), 60.0) + self.assertEqual(env.k.network.get_max_speed("bottom", 1), 60.0) + self.assertEqual(env.k.network.get_max_speed("bottom", 2), 60.0) + + env.k.network.set_max_speed("bottom", 30.0) + env.step(None) + self.assertEqual(env.k.network.get_max_speed("bottom", 0), 30.0) + self.assertEqual(env.k.network.get_max_speed("bottom", 1), 30.0) + self.assertEqual(env.k.network.get_max_speed("bottom", 2), 30.0) + + if __name__ == '__main__': unittest.main() diff --git a/tests/fast_tests/test_scenarios.py b/tests/fast_tests/test_scenarios.py index 5fccdcb3b..8778b54ba 100644 --- a/tests/fast_tests/test_scenarios.py +++ b/tests/fast_tests/test_scenarios.py @@ -300,7 +300,13 @@ def test_specify_routes(self): ) self.assertEqual( - ['119257914'], + ['119257908#0', + '119257908#1', + '119257908#1-AddedOffRampEdge', + '119257908#1-AddedOnRampEdge', + '119257908#2', + '119257908#3', + '119257914'], sorted(list(network.specify_routes(network.net_params).keys())) ) @@ -357,7 +363,14 @@ def test_specify_routes(self): ) self.assertEqual( - ['119257914', 'ghost0'], + ['119257908#0', + '119257908#1', + '119257908#1-AddedOffRampEdge', + '119257908#1-AddedOnRampEdge', + '119257908#2', + '119257908#3', + '119257914', + 'ghost0'], sorted(list(network.specify_routes(network.net_params).keys())) ) diff --git a/tests/fast_tests/test_vehicles.py b/tests/fast_tests/test_vehicles.py index 7e1405007..6bd79bb06 100644 --- a/tests/fast_tests/test_vehicles.py +++ b/tests/fast_tests/test_vehicles.py @@ -2,13 +2,16 @@ import os import numpy as np -from flow.core.params import VehicleParams +from flow.core.params import VehicleParams, EnvParams from flow.core.params import SumoCarFollowingParams, NetParams, \ InitialConfig, SumoParams, SumoLaneChangeParams from flow.controllers.car_following_models import IDMController, \ SimCarFollowingController from flow.controllers.lane_change_controllers import StaticLaneChanger from flow.controllers.rlcontroller import RLController +from flow.networks import RingNetwork +from flow.envs import TestEnv +from flow.utils.registry import make_create_env from tests.setup_scripts import ring_road_exp_setup, highway_exp_setup @@ -195,6 +198,99 @@ def test_remove(self): self.assertEqual(env.k.vehicle.num_rl_vehicles, len(env.k.vehicle.get_rl_ids())) + def test_set_vehicle_type(self): + """Validate the functionality of the set_vehicle_type method. + + This test initializes a network with a human-driven vehicle and then + replaces it with an RL vehicles, and tests various internal parameters + before and after. + """ + # Create the flow_params object, with types for both human and RL + # vehicles. + sim_params = SumoParams(sim_step=0.1, render=False) + + vehicles = VehicleParams() + vehicles.add( + veh_id="human", + acceleration_controller=(IDMController, {}), + car_following_params=SumoCarFollowingParams( + speed_mode="aggressive", + ), + num_vehicles=1, + ) + vehicles.add( + veh_id="rl", + acceleration_controller=(RLController, {}), + car_following_params=SumoCarFollowingParams( + speed_mode="aggressive", + ), + num_vehicles=0, + ) + + env_params = EnvParams( + horizon=20, + additional_params={ + "target_velocity": 8, + "max_accel": 1, + "max_decel": 1, + "sort_vehicles": False, + } + ) + + net_params = NetParams(additional_params={ + "length": 230, + "lanes": 1, + "speed_limit": 30, + "resolution": 40 + }) + + flow_params = dict( + exp_tag="RingRoadTest", + env_name=TestEnv, + network=RingNetwork, + simulator='traci', + sim=sim_params, + env=env_params, + net=net_params, + veh=vehicles, + ) + + # Create the environment. + create_env, _ = make_create_env(flow_params) + env = create_env() + env.reset() + + # Check the vehicle attributes. + self.assertEqual(env.k.vehicle.num_vehicles, 1) + self.assertEqual(env.k.vehicle.num_rl_vehicles, 0) + self.assertEqual(env.k.vehicle.get_ids(), ["human_0"]) + self.assertEqual(env.k.vehicle.get_human_ids(), ["human_0"]) + self.assertEqual(env.k.vehicle.get_rl_ids(), []) + self.assertTrue(isinstance( + env.k.vehicle.get_acc_controller("human_0"), IDMController)) + + # Update the vehicle type. + env.k.vehicle.set_vehicle_type("human_0", "rl") + + # Check the vehicle attributes after updating. + self.assertEqual(env.k.vehicle.num_vehicles, 1) + self.assertEqual(env.k.vehicle.num_rl_vehicles, 1) + self.assertEqual(env.k.vehicle.get_ids(), ["human_0"]) + self.assertEqual(env.k.vehicle.get_human_ids(), []) + self.assertEqual(env.k.vehicle.get_rl_ids(), ["human_0"]) + self.assertTrue(isinstance( + env.k.vehicle.get_acc_controller("human_0"), RLController)) + + # Update the vehicle type to check the other direction + env.k.vehicle.set_vehicle_type("human_0", "human") + self.assertEqual(env.k.vehicle.num_vehicles, 1) + self.assertEqual(env.k.vehicle.num_rl_vehicles, 0) + self.assertEqual(env.k.vehicle.get_ids(), ["human_0"]) + self.assertEqual(env.k.vehicle.get_human_ids(), ["human_0"]) + self.assertEqual(env.k.vehicle.get_rl_ids(), []) + self.assertTrue(isinstance( + env.k.vehicle.get_acc_controller("human_0"), IDMController)) + class TestMultiLaneData(unittest.TestCase): """ diff --git a/tests/fast_tests/test_visualizers.py b/tests/fast_tests/test_visualizers.py index d2f4a20a4..659734ba9 100644 --- a/tests/fast_tests/test_visualizers.py +++ b/tests/fast_tests/test_visualizers.py @@ -1,5 +1,3 @@ -from flow.visualize import visualizer_rllib as vs_rllib -from flow.visualize.visualizer_rllib import visualizer_rllib import flow.visualize.capacity_diagram_generator as cdg import flow.visualize.time_space_diagram as tsd import flow.visualize.plot_ray_results as prr @@ -14,45 +12,6 @@ os.environ['TEST_FLAG'] = 'True' -class TestVisualizerRLlib(unittest.TestCase): - """Tests visualizer_rllib""" - - def test_visualizer_single(self): - """Test for single agent""" - try: - ray.init(num_cpus=1) - except Exception: - pass - # current path - current_path = os.path.realpath(__file__).rsplit('/', 1)[0] - - # run the experiment and check it doesn't crash - arg_str = '{}/../data/rllib_data/single_agent 1 --num_rollouts 1 ' \ - '--render_mode no_render ' \ - '--horizon 10'.format(current_path).split() - parser = vs_rllib.create_parser() - pass_args = parser.parse_args(arg_str) - visualizer_rllib(pass_args) - - # FIXME(ev) set the horizon so that this runs faster - def test_visualizer_multi(self): - """Test for multi-agent visualization""" - try: - ray.init(num_cpus=1) - except Exception: - pass - # current path - current_path = os.path.realpath(__file__).rsplit('/', 1)[0] - - # run the experiment and check it doesn't crash - arg_str = '{}/../data/rllib_data/multi_agent 1 --num_rollouts 1 ' \ - '--render_mode no_render ' \ - '--horizon 10'.format(current_path).split() - parser = vs_rllib.create_parser() - pass_args = parser.parse_args(arg_str) - visualizer_rllib(pass_args) - - class TestPlotters(unittest.TestCase): def test_capacity_diagram_generator(self): @@ -94,54 +53,54 @@ def test_time_space_diagram_figure_eight(self): dir_path = os.path.dirname(os.path.realpath(__file__)) flow_params = tsd.get_flow_params( os.path.join(dir_path, 'test_files/fig8.json')) - emission_data = tsd.import_data_from_trajectory( + emission_data, _, _, _ = tsd.import_data_from_trajectory( os.path.join(dir_path, 'test_files/fig8_emission.csv'), flow_params) - segs, _ = tsd.get_time_space_data(emission_data, flow_params) + segs, _ = tsd.get_time_space_data(emission_data, flow_params['network']) expected_segs = np.array([ - [[1., 60.], [2., 59.]], - [[2., 59.], [3., 57.02]], - [[3., 57.02], [4., 54.05]], - [[1., 23.8], [2., 22.81]], - [[2., 22.81], [3., 20.83]], - [[3., 20.83], [4., 17.89]], - [[1., 182.84166941], [2., 181.85166941]], - [[2., 181.85166941], [3., 179.87166941]], - [[3., 179.87166941], [4., 176.92166941]], - [[1., 154.07166941], [2., 153.08166941]], - [[2., 153.08166941], [3., 151.10166941]], - [[3., 151.10166941], [4., 148.16166941]], - [[1., 125.30166941], [2., 124.31166941]], - [[2., 124.31166941], [3., 122.34166941]], - [[3., 122.34166941], [4., 119.39166941]], - [[1., 96.54166941], [2., 95.54166941]], - [[2., 95.54166941], [3., 93.56166941]], - [[3., 93.56166941], [4., 90.59166941]], - [[1., -203.16166941], [2., -202.17166941]], - [[2., -202.17166941], [3., -200.02166941]], - [[3., -200.02166941], [4., -197.07166941]], - [[1., -174.40166941], [2., -173.40166941]], - [[2., -173.40166941], [3., -171.43166941]], - [[3., -171.43166941], [4., -168.48166941]], - [[1., -145.63166941], [2., -144.64166941]], - [[2., -144.64166941], [3., -142.66166941]], - [[3., -142.66166941], [4., -139.72166941]], - [[1., -116.86166941], [2., -115.87166941]], - [[2., -115.87166941], [3., -113.89166941]], - [[3., -113.89166941], [4., -110.95166941]], - [[1., -88.09166941], [2., -87.10166941]], - [[2., -87.10166941], [3., -85.13166941]], - [[3., -85.13166941], [4., -82.18166941]], - [[1., -59.33], [2., -58.34]], - [[2., -58.34], [3., -56.36]], - [[3., -56.36], [4., -53.42]], - [[1., -30.56], [2., -29.72]], - [[2., -29.72], [3., -27.97]], - [[3., -27.97], [4., -25.22]], - [[1., -1.79], [2., -0.8]], - [[2., -0.8], [3., 208.64166941]], - [[3., 208.64166941], [4., 205.69166941]]] + [[1., 263.16166941], [2., 262.16166941]], + [[2., 262.16166941], [3., 260.18166941]], + [[3., 260.18166941], [4., 257.21166941]], + [[1., 226.96166941], [2., 225.97166941]], + [[2., 225.97166941], [3., 223.99166941]], + [[3., 223.99166941], [4., 221.05166941]], + [[1., 386.00333882], [2., 385.01333882]], + [[2., 385.01333882], [3., 383.03333882]], + [[3., 383.03333882], [4., 380.08333882]], + [[1., 357.23333882], [2., 356.24333882]], + [[2., 356.24333882], [3., 354.26333882]], + [[3., 354.26333882], [4., 351.32333882]], + [[1., 328.46333882], [2., 327.47333882]], + [[2., 327.47333882], [3., 325.50333882]], + [[3., 325.50333882], [4., 322.55333882]], + [[1., 299.70333882], [2., 298.70333882]], + [[2., 298.70333882], [3., 296.72333882]], + [[3., 296.72333882], [4., 293.75333882]], + [[1., 0.], [2., 0.99]], + [[2., 0.99], [3., 3.14]], + [[3., 3.14], [4., 6.09]], + [[1., 28.76], [2., 29.76]], + [[2., 29.76], [3., 31.73]], + [[3., 31.73], [4., 34.68]], + [[1., 57.53], [2., 58.52]], + [[2., 58.52], [3., 60.5]], + [[3., 60.5], [4., 63.44]], + [[1., 86.3], [2., 87.29]], + [[2., 87.29], [3., 89.27]], + [[3., 89.27], [4., 92.21]], + [[1., 115.07], [2., 116.06]], + [[2., 116.06], [3., 118.03]], + [[3., 118.03], [4., 120.98]], + [[1., 143.83166941], [2., 144.82166941]], + [[2., 144.82166941], [3., 146.80166941]], + [[3., 146.80166941], [4., 149.74166941]], + [[1., 172.60166941], [2., 173.44166941]], + [[2., 173.44166941], [3., 175.19166941]], + [[3., 175.19166941], [4., 177.94166941]], + [[1., 201.37166941], [2., 202.36166941]], + [[2., 202.36166941], [3., 411.80333882]], + [[3., 411.80333882], [4., 408.85333882]]] ) np.testing.assert_array_almost_equal(segs, expected_segs) @@ -150,16 +109,16 @@ def test_time_space_diagram_merge(self): dir_path = os.path.dirname(os.path.realpath(__file__)) flow_params = tsd.get_flow_params( os.path.join(dir_path, 'test_files/merge.json')) - emission_data = tsd.import_data_from_trajectory( + emission_data, _, _, _ = tsd.import_data_from_trajectory( os.path.join(dir_path, 'test_files/merge_emission.csv'), flow_params) - segs, _ = tsd.get_time_space_data(emission_data, flow_params) + segs, _ = tsd.get_time_space_data(emission_data, flow_params['network']) expected_segs = np.array([ - [[2.0000e-01, 7.2949e+02], [4.0000e-01, 7.2953e+02]], - [[4.0000e-01, 7.2953e+02], [6.0000e-01, 7.2961e+02]], - [[6.0000e-01, 7.2961e+02], [8.0000e-01, 7.2973e+02]], - [[8.0000e-01, 7.2973e+02], [1.0000e+00, 7.2988e+02]]] + [[2.0000e-01, 7.2463e+02], [4.0000e-01, 7.2467e+02]], + [[4.0000e-01, 7.2467e+02], [6.0000e-01, 7.2475e+02]], + [[6.0000e-01, 7.2475e+02], [8.0000e-01, 7.2487e+02]], + [[8.0000e-01, 7.2487e+02], [1.0000e+00, 7.2502e+02]]] ) np.testing.assert_array_almost_equal(segs, expected_segs) @@ -168,37 +127,37 @@ def test_time_space_diagram_I210(self): dir_path = os.path.dirname(os.path.realpath(__file__)) module = __import__("examples.exp_configs.non_rl", fromlist=["i210_subnetwork"]) flow_params = getattr(module, "i210_subnetwork").flow_params - emission_data = tsd.import_data_from_trajectory( + emission_data, _, _, _ = tsd.import_data_from_trajectory( os.path.join(dir_path, 'test_files/i210_emission.csv'), flow_params) - segs, _ = tsd.get_time_space_data(emission_data, flow_params) + segs, _ = tsd.get_time_space_data(emission_data, flow_params['network']) expected_segs = { - 1: np.array([ - [[0.8, 5.1], [1.6, 23.37]], - [[1.6, 23.37], [2.4, 42.02]], - [[2.4, 42.02], [3.2, 61.21]], - [[3.2, 61.21], [4., 18.87]], - [[4., 18.87], [4.8, 39.93]], - [[2.4, 5.1], [3.2, 22.97]], - [[3.2, 22.97], [4., 40.73]]] - ), 2: np.array([ - [[2.4, 5.1], [3.2, 23.98]], - [[3.2, 23.98], [4., 43.18]]] + [[-719.2, 3.77], [-718.4, 22.04]], + [[-718.4, 22.04], [-717.6, 40.69]], + [[-717.6, 40.69], [-716.8, 59.88]], + [[-716.8, 59.88], [-716., 17.54]], + [[-716., 17.54], [-715.2, 38.6]], + [[-717.6, 3.77], [-716.8, 21.64]], + [[-716.8, 21.64], [-716., 39.4]]] ), 3: np.array([ - [[0.8, 5.1], [1.6, 23.72]], - [[1.6, 23.72], [2.4, 43.06]], - [[2.4, 43.06], [3.2, 1.33]], - [[3.2, 1.33], [4., 21.65]], - [[4., 21.65], [4.8, 43.46]], - [[2.4, 5.1], [3.2, 23.74]], - [[3.2, 23.74], [4., 42.38]]] + [[-717.6, 3.77], [-716.8, 22.65]], + [[-716.8, 22.65], [-716., 41.85]]] ), 4: np.array([ - [[2.4, 5.1], [3.2, 23.6]], - [[3.2, 23.6], [4., 42.46]]] + [[-719.2, 3.77], [-718.4, 22.39]], + [[-718.4, 22.39], [-717.6, 41.73]], + [[-717.6, 41.73], [-716.8, 0.]], + [[-716.8, 0.], [-716., 20.32]], + [[-716., 20.32], [-715.2, 42.13]], + [[-717.6, 3.77], [-716.8, 22.41]], + [[-716.8, 22.41], [-716., 41.05]]] + ), + 5: np.array([ + [[-717.6, 3.77], [-716.8, 22.27]], + [[-716.8, 22.27], [-716., 41.13]]] )} for lane, expected_seg in expected_segs.items(): @@ -208,106 +167,30 @@ def test_time_space_diagram_ring_road(self): dir_path = os.path.dirname(os.path.realpath(__file__)) flow_params = tsd.get_flow_params( os.path.join(dir_path, 'test_files/ring_230.json')) - emission_data = tsd.import_data_from_trajectory( + emission_data, _, _, _ = tsd.import_data_from_trajectory( os.path.join(dir_path, 'test_files/ring_230_emission.csv'), flow_params) - segs, _ = tsd.get_time_space_data(emission_data, flow_params) + segs, _ = tsd.get_time_space_data(emission_data, flow_params['network']) expected_segs = np.array([ - [[1.0000e-01, 0.0000e+00], [2.0000e-01, 1.0000e-02]], - [[2.0000e-01, 1.0000e-02], [3.0000e-01, 2.0000e-02]], - [[3.0000e-01, 2.0000e-02], [4.0000e-01, 5.0000e-02]], - [[4.0000e-01, 5.0000e-02], [5.0000e-01, 8.0000e-02]], - [[5.0000e-01, 8.0000e-02], [6.0000e-01, 1.2000e-01]], - [[1.0000e-01, 9.5500e+00], [2.0000e-01, 9.5500e+00]], - [[2.0000e-01, 9.5500e+00], [3.0000e-01, 9.5700e+00]], - [[3.0000e-01, 9.5700e+00], [4.0000e-01, 9.5900e+00]], - [[4.0000e-01, 9.5900e+00], [5.0000e-01, 9.6200e+00]], - [[5.0000e-01, 9.6200e+00], [6.0000e-01, 9.6600e+00]], - [[1.0000e-01, 9.5550e+01], [2.0000e-01, 9.5560e+01]], - [[2.0000e-01, 9.5560e+01], [3.0000e-01, 9.5580e+01]], - [[3.0000e-01, 9.5580e+01], [4.0000e-01, 9.5600e+01]], - [[4.0000e-01, 9.5600e+01], [5.0000e-01, 9.5630e+01]], - [[5.0000e-01, 9.5630e+01], [6.0000e-01, 9.5670e+01]], - [[1.0000e-01, 1.0510e+02], [2.0000e-01, 1.0511e+02]], - [[2.0000e-01, 1.0511e+02], [3.0000e-01, 1.0512e+02]], - [[3.0000e-01, 1.0512e+02], [4.0000e-01, 1.0515e+02]], - [[4.0000e-01, 1.0515e+02], [5.0000e-01, 1.0518e+02]], - [[5.0000e-01, 1.0518e+02], [6.0000e-01, 1.0522e+02]], - [[1.0000e-01, 1.1465e+02], [2.0000e-01, 1.1465e+02]], - [[2.0000e-01, 1.1465e+02], [3.0000e-01, 1.1467e+02]], - [[3.0000e-01, 1.1467e+02], [4.0000e-01, 1.1469e+02]], - [[4.0000e-01, 1.1469e+02], [5.0000e-01, 1.1472e+02]], - [[5.0000e-01, 1.1472e+02], [6.0000e-01, 1.1476e+02]], - [[1.0000e-01, 1.2429e+02], [2.0000e-01, 1.2430e+02]], - [[2.0000e-01, 1.2430e+02], [3.0000e-01, 1.2431e+02]], - [[3.0000e-01, 1.2431e+02], [4.0000e-01, 1.2434e+02]], - [[4.0000e-01, 1.2434e+02], [5.0000e-01, 1.2437e+02]], - [[5.0000e-01, 1.2437e+02], [6.0000e-01, 1.2441e+02]], - [[1.0000e-01, 1.3384e+02], [2.0000e-01, 1.3384e+02]], - [[2.0000e-01, 1.3384e+02], [3.0000e-01, 1.3386e+02]], - [[3.0000e-01, 1.3386e+02], [4.0000e-01, 1.3388e+02]], - [[4.0000e-01, 1.3388e+02], [5.0000e-01, 1.3391e+02]], - [[1.0000e-01, 1.4338e+02], [2.0000e-01, 1.4339e+02]], - [[2.0000e-01, 1.4339e+02], [3.0000e-01, 1.4341e+02]], - [[3.0000e-01, 1.4341e+02], [4.0000e-01, 1.4343e+02]], - [[4.0000e-01, 1.4343e+02], [5.0000e-01, 1.4346e+02]], - [[1.0000e-01, 1.5293e+02], [2.0000e-01, 1.5294e+02]], - [[2.0000e-01, 1.5294e+02], [3.0000e-01, 1.5295e+02]], - [[3.0000e-01, 1.5295e+02], [4.0000e-01, 1.5297e+02]], - [[4.0000e-01, 1.5297e+02], [5.0000e-01, 1.5301e+02]], - [[1.0000e-01, 1.6247e+02], [2.0000e-01, 1.6248e+02]], - [[2.0000e-01, 1.6248e+02], [3.0000e-01, 1.6250e+02]], - [[3.0000e-01, 1.6250e+02], [4.0000e-01, 1.6252e+02]], - [[4.0000e-01, 1.6252e+02], [5.0000e-01, 1.6255e+02]], - [[1.0000e-01, 1.7202e+02], [2.0000e-01, 1.7203e+02]], - [[2.0000e-01, 1.7203e+02], [3.0000e-01, 1.7204e+02]], - [[3.0000e-01, 1.7204e+02], [4.0000e-01, 1.7207e+02]], - [[4.0000e-01, 1.7207e+02], [5.0000e-01, 1.7210e+02]], - [[1.0000e-01, 1.8166e+02], [2.0000e-01, 1.8167e+02]], - [[2.0000e-01, 1.8167e+02], [3.0000e-01, 1.8169e+02]], - [[3.0000e-01, 1.8169e+02], [4.0000e-01, 1.8171e+02]], - [[4.0000e-01, 1.8171e+02], [5.0000e-01, 1.8174e+02]], - [[1.0000e-01, 1.9090e+01], [2.0000e-01, 1.9100e+01]], - [[2.0000e-01, 1.9100e+01], [3.0000e-01, 1.9110e+01]], - [[3.0000e-01, 1.9110e+01], [4.0000e-01, 1.9140e+01]], - [[4.0000e-01, 1.9140e+01], [5.0000e-01, 1.9170e+01]], - [[1.0000e-01, 1.9121e+02], [2.0000e-01, 1.9122e+02]], - [[2.0000e-01, 1.9122e+02], [3.0000e-01, 1.9123e+02]], - [[3.0000e-01, 1.9123e+02], [4.0000e-01, 1.9126e+02]], - [[4.0000e-01, 1.9126e+02], [5.0000e-01, 1.9129e+02]], - [[1.0000e-01, 2.0075e+02], [2.0000e-01, 2.0076e+02]], - [[2.0000e-01, 2.0076e+02], [3.0000e-01, 2.0078e+02]], - [[3.0000e-01, 2.0078e+02], [4.0000e-01, 2.0081e+02]], - [[4.0000e-01, 2.0081e+02], [5.0000e-01, 2.0085e+02]], - [[1.0000e-01, 2.8640e+01], [2.0000e-01, 2.8640e+01]], - [[2.0000e-01, 2.8640e+01], [3.0000e-01, 2.8660e+01]], - [[3.0000e-01, 2.8660e+01], [4.0000e-01, 2.8680e+01]], - [[4.0000e-01, 2.8680e+01], [5.0000e-01, 2.8710e+01]], - [[1.0000e-01, 3.8180e+01], [2.0000e-01, 3.8190e+01]], - [[2.0000e-01, 3.8190e+01], [3.0000e-01, 3.8210e+01]], - [[3.0000e-01, 3.8210e+01], [4.0000e-01, 3.8230e+01]], - [[4.0000e-01, 3.8230e+01], [5.0000e-01, 3.8260e+01]], - [[1.0000e-01, 4.7730e+01], [2.0000e-01, 4.7740e+01]], - [[2.0000e-01, 4.7740e+01], [3.0000e-01, 4.7750e+01]], - [[3.0000e-01, 4.7750e+01], [4.0000e-01, 4.7770e+01]], - [[4.0000e-01, 4.7770e+01], [5.0000e-01, 4.7810e+01]], - [[1.0000e-01, 5.7270e+01], [2.0000e-01, 5.7280e+01]], - [[2.0000e-01, 5.7280e+01], [3.0000e-01, 5.7300e+01]], - [[3.0000e-01, 5.7300e+01], [4.0000e-01, 5.7320e+01]], - [[4.0000e-01, 5.7320e+01], [5.0000e-01, 5.7350e+01]], - [[1.0000e-01, 6.6920e+01], [2.0000e-01, 6.6930e+01]], - [[2.0000e-01, 6.6930e+01], [3.0000e-01, 6.6940e+01]], - [[3.0000e-01, 6.6940e+01], [4.0000e-01, 6.6970e+01]], - [[4.0000e-01, 6.6970e+01], [5.0000e-01, 6.7000e+01]], - [[1.0000e-01, 7.6460e+01], [2.0000e-01, 7.6470e+01]], - [[2.0000e-01, 7.6470e+01], [3.0000e-01, 7.6490e+01]], - [[3.0000e-01, 7.6490e+01], [4.0000e-01, 7.6510e+01]], - [[4.0000e-01, 7.6510e+01], [5.0000e-01, 7.6540e+01]], - [[1.0000e-01, 8.6010e+01], [2.0000e-01, 8.6020e+01]], - [[2.0000e-01, 8.6020e+01], [3.0000e-01, 8.6030e+01]], - [[3.0000e-01, 8.6030e+01], [4.0000e-01, 8.6060e+01]], - [[4.0000e-01, 8.6060e+01], [5.0000e-01, 8.6090e+01]]] + [[-7.50000000e+01, 0.00000000e+00], [-7.49000000e+01, 7.98415842e-03]], + [[-7.49000000e+01, 7.98415842e-03], [-7.48000000e+01, 2.37963776e-02]], + [[-7.48000000e+01, 2.37963776e-02], [-7.47000000e+01, 4.72776801e-02]], + [[-7.50000000e+01, 9.54545455e+00], [-7.49000000e+01, 9.55343870e+00]], + [[-7.49000000e+01, 9.55343870e+00], [-7.48000000e+01, 9.56925092e+00]], + [[-7.48000000e+01, 9.56925092e+00], [-7.47000000e+01, 9.59273223e+00]], + [[-7.50000000e+01, 1.90909091e+01], [-7.49000000e+01, 1.90988932e+01]], + [[-7.49000000e+01, 1.90988932e+01], [-7.48000000e+01, 1.91147055e+01]], + [[-7.48000000e+01, 1.91147055e+01], [-7.47000000e+01, 1.91381868e+01]], + [[-7.50000000e+01, 2.86363636e+01], [-7.49000000e+01, 2.86443478e+01]], + [[-7.49000000e+01, 2.86443478e+01], [-7.48000000e+01, 2.86601600e+01]], + [[-7.48000000e+01, 2.86601600e+01], [-7.47000000e+01, 2.86836413e+01]], + [[-7.50000000e+01, 3.81818182e+01], [-7.49000000e+01, 3.81898023e+01]], + [[-7.49000000e+01, 3.81898023e+01], [-7.48000000e+01, 3.82056146e+01]], + [[-7.48000000e+01, 3.82056146e+01], [-7.47000000e+01, 3.82290959e+01]], + [[-7.50000000e+01, 4.77272727e+01], [-7.49000000e+01, 4.77352569e+01]], + [[-7.49000000e+01, 4.77352569e+01], [-7.48000000e+01, 4.77510691e+01]], + [[-7.48000000e+01, 4.77510691e+01], [-7.47000000e+01, 4.77745504e+01]]] ) np.testing.assert_array_almost_equal(segs, expected_segs) diff --git a/tests/slow_tests/test_benchmarks.py b/tests/slow_tests/test_benchmarks.py index c119d4bd8..4a50e84cb 100644 --- a/tests/slow_tests/test_benchmarks.py +++ b/tests/slow_tests/test_benchmarks.py @@ -84,8 +84,7 @@ def ray_runner(self, num_runs, flow_params, version): # Register as rllib env register_env(env_name, create_env) - - alg = ppo.PPOAgent( + alg = ppo.ppo.PPOTrainer( env=env_name, config=config) for i in range(num_runs):