experiment results

danielbinschmid · Feb 12, 2024 · a9c3538 · a9c3538
1 parent ad6986a
commit a9c3538
Show file tree

Hide file tree

Showing 1,306 changed files with 17 additions and 7 deletions.
diff --git a/runnables/test_suite_eval/eval_rl.sh b/runnables/test_suite_eval/eval_rl.sh
@@ -1,8 +1,8 @@
 #!/bin/bash
 
-for level in {5..20}
+for level in {14,17,20,23,26}
 do
-  python C_trajectory_following.py --output_folder "../../checkpointed_models/" --eval_set "./eval-v0_n-ctrl-points-3_n-tracks-1000_2024-02-12_10:59:21_d689e67e-4179-4764-a474-e5f3237a239d" --discr_level $level
-  python pid.py --discr_level $level --eval_set "./eval-v0_n-ctrl-points-3_n-tracks-1000_2024-02-12_10:59:21_d689e67e-4179-4764-a474-e5f3237a239d"
+  # python C_trajectory_following.py --output_folder "../../checkpointed_models/" --eval_set "./eval-v0_n-ctrl-points-10_n-tracks-200_2024-02-12_15:01:53_bdee17ed-07ad-4e26-8e68-fa0a5ca1318c" --discr_level $level
+  python pid.py --discr_level $level --eval_set "./eval-v0_n-ctrl-points-10_n-tracks-200_2024-02-12_15:01:53_bdee17ed-07ad-4e26-8e68-fa0a5ca1318c"
 
 done
diff --git a/runnables/test_suite_eval/eval_tracks.py b/runnables/test_suite_eval/eval_tracks.py
@@ -22,8 +22,8 @@ def gen_eval_tracks(target_folder: str, n_tracks: int, n_ctrl_points: int=3):
 
     for i in tqdm(range(n_tracks)):
         _, ctrl_wp = TrajectoryFactory.gen_random_trajectory(
-            start=np.array([0, 0, 0]),
-            n_discr_level=10,
+            start=np.array([0, 0, 1]),
+            n_discr_level=20,
             n_ctrl_points=n_ctrl_points,
             std_dev_deg=50,
             distance_between_ctrl_points=1.3,
@@ -60,8 +60,8 @@ def load_eval_tracks(folder: str, discr_level: int=10) -> List[DiscretizedTrajec
 
     if gen:
         t_folder = "./eval-v0"
-        n_tracks = 1000
-        gen_eval_tracks(t_folder, n_tracks=n_tracks)
+        n_tracks = 200
+        gen_eval_tracks(t_folder, n_tracks=n_tracks, n_ctrl_points=10)
     if load: 
         tracks = load_eval_tracks(load_folder)
         for track in tracks:

diff --git a/runnables/test_suite_eval/long_trajectory_jsons/pid_14.json b/runnables/test_suite_eval/long_trajectory_jsons/pid_14.json
@@ -0,0 +1 @@
+{"success_rate": 0.24812030075187969, "avg mean dev": 0.12312815303888709, "avg max dev": 0.46306573291494524, "avt time": 25.69886363636364}
diff --git a/runnables/test_suite_eval/long_trajectory_jsons/pid_17.json b/runnables/test_suite_eval/long_trajectory_jsons/pid_17.json
@@ -0,0 +1 @@
+{"success_rate": 0.36507936507936506, "avg mean dev": 0.10840245515779884, "avg max dev": 0.36533508928277664, "avt time": 26.56811594202898}
diff --git a/runnables/test_suite_eval/long_trajectory_jsons/pid_20.json b/runnables/test_suite_eval/long_trajectory_jsons/pid_20.json
@@ -0,0 +1 @@
+{"success_rate": 0.4791666666666667, "avg mean dev": 0.09615903659247474, "avg max dev": 0.28994711420539, "avt time": 26.77921195652174}
diff --git a/runnables/test_suite_eval/long_trajectory_jsons/pid_23.json b/runnables/test_suite_eval/long_trajectory_jsons/pid_23.json
@@ -0,0 +1 @@
+{"success_rate": 0.49748743718592964, "avg mean dev": 0.08853108014915095, "avg max dev": 0.22978509050899154, "avt time": 27.1082702020202}
diff --git a/runnables/test_suite_eval/long_trajectory_jsons/pid_26.json b/runnables/test_suite_eval/long_trajectory_jsons/pid_26.json
@@ -0,0 +1 @@
+{"success_rate": 0.5, "avg mean dev": 0.08308981149934497, "avg max dev": 0.20903981543228453, "avt time": 28.1221875}
diff --git a/runnables/test_suite_eval/long_trajectory_jsons/rl_14.json b/runnables/test_suite_eval/long_trajectory_jsons/rl_14.json
@@ -0,0 +1 @@
+{"success_rate": 0.985, "avg mean dev": 0.1772167915118993, "avg max dev": 0.4917212704683174, "avt time": 14.34890016920474}
diff --git a/runnables/test_suite_eval/long_trajectory_jsons/rl_17.json b/runnables/test_suite_eval/long_trajectory_jsons/rl_17.json
@@ -0,0 +1 @@
+{"success_rate": 0.99, "avg mean dev": 0.1433386324227451, "avg max dev": 0.392061174557603, "avt time": 16.289898989898987}
diff --git a/runnables/test_suite_eval/long_trajectory_jsons/rl_20.json b/runnables/test_suite_eval/long_trajectory_jsons/rl_20.json
@@ -0,0 +1 @@
+{"success_rate": 0.99, "avg mean dev": 0.11955849172075408, "avg max dev": 0.313560755400775, "avt time": 18.615319865319865}
diff --git a/runnables/test_suite_eval/long_trajectory_jsons/rl_23.json b/runnables/test_suite_eval/long_trajectory_jsons/rl_23.json
@@ -0,0 +1 @@
+{"success_rate": 0.995, "avg mean dev": 0.10246450706590106, "avg max dev": 0.27034303375162694, "avt time": 20.263651591289783}
diff --git a/runnables/test_suite_eval/long_trajectory_jsons/rl_26.json b/runnables/test_suite_eval/long_trajectory_jsons/rl_26.json
@@ -0,0 +1 @@
+{"success_rate": 0.99, "avg mean dev": 0.09032995407396031, "avg max dev": 0.22982543933962724, "avt time": 22.31952861952862}
diff --git a/runnables/test_suite_eval/pos_logs/0.npy b/runnables/test_suite_eval/pos_logs/0.npy
diff --git a/runnables/test_suite_eval/pos_logs/1.npy b/runnables/test_suite_eval/pos_logs/1.npy
diff --git a/runnables/test_suite_eval/pos_logs/10.npy b/runnables/test_suite_eval/pos_logs/10.npy
diff --git a/runnables/test_suite_eval/pos_logs/100.npy b/runnables/test_suite_eval/pos_logs/100.npy
diff --git a/runnables/test_suite_eval/pos_logs/1000.npy b/runnables/test_suite_eval/pos_logs/1000.npy
diff --git a/runnables/test_suite_eval/pos_logs/1001.npy b/runnables/test_suite_eval/pos_logs/1001.npy
diff --git a/runnables/test_suite_eval/pos_logs/1002.npy b/runnables/test_suite_eval/pos_logs/1002.npy
diff --git a/runnables/test_suite_eval/pos_logs/1003.npy b/runnables/test_suite_eval/pos_logs/1003.npy
diff --git a/runnables/test_suite_eval/pos_logs/1004.npy b/runnables/test_suite_eval/pos_logs/1004.npy
diff --git a/runnables/test_suite_eval/pos_logs/1005.npy b/runnables/test_suite_eval/pos_logs/1005.npy
diff --git a/runnables/test_suite_eval/pos_logs/1006.npy b/runnables/test_suite_eval/pos_logs/1006.npy
diff --git a/runnables/test_suite_eval/pos_logs/1007.npy b/runnables/test_suite_eval/pos_logs/1007.npy
diff --git a/runnables/test_suite_eval/pos_logs/1008.npy b/runnables/test_suite_eval/pos_logs/1008.npy
diff --git a/runnables/test_suite_eval/pos_logs/1009.npy b/runnables/test_suite_eval/pos_logs/1009.npy
diff --git a/runnables/test_suite_eval/pos_logs/101.npy b/runnables/test_suite_eval/pos_logs/101.npy
diff --git a/runnables/test_suite_eval/pos_logs/1010.npy b/runnables/test_suite_eval/pos_logs/1010.npy
diff --git a/runnables/test_suite_eval/pos_logs/1011.npy b/runnables/test_suite_eval/pos_logs/1011.npy
diff --git a/runnables/test_suite_eval/pos_logs/1012.npy b/runnables/test_suite_eval/pos_logs/1012.npy
diff --git a/runnables/test_suite_eval/pos_logs/1013.npy b/runnables/test_suite_eval/pos_logs/1013.npy
diff --git a/runnables/test_suite_eval/pos_logs/1014.npy b/runnables/test_suite_eval/pos_logs/1014.npy
diff --git a/runnables/test_suite_eval/pos_logs/1015.npy b/runnables/test_suite_eval/pos_logs/1015.npy
diff --git a/runnables/test_suite_eval/pos_logs/1016.npy b/runnables/test_suite_eval/pos_logs/1016.npy
diff --git a/runnables/test_suite_eval/pos_logs/1017.npy b/runnables/test_suite_eval/pos_logs/1017.npy
diff --git a/runnables/test_suite_eval/pos_logs/1018.npy b/runnables/test_suite_eval/pos_logs/1018.npy
diff --git a/runnables/test_suite_eval/pos_logs/1019.npy b/runnables/test_suite_eval/pos_logs/1019.npy
diff --git a/runnables/test_suite_eval/pos_logs/102.npy b/runnables/test_suite_eval/pos_logs/102.npy
diff --git a/runnables/test_suite_eval/pos_logs/1020.npy b/runnables/test_suite_eval/pos_logs/1020.npy
diff --git a/runnables/test_suite_eval/pos_logs/1021.npy b/runnables/test_suite_eval/pos_logs/1021.npy
diff --git a/runnables/test_suite_eval/pos_logs/1022.npy b/runnables/test_suite_eval/pos_logs/1022.npy
diff --git a/runnables/test_suite_eval/pos_logs/1023.npy b/runnables/test_suite_eval/pos_logs/1023.npy
diff --git a/runnables/test_suite_eval/pos_logs/1024.npy b/runnables/test_suite_eval/pos_logs/1024.npy
diff --git a/runnables/test_suite_eval/pos_logs/1025.npy b/runnables/test_suite_eval/pos_logs/1025.npy
diff --git a/runnables/test_suite_eval/pos_logs/1026.npy b/runnables/test_suite_eval/pos_logs/1026.npy
diff --git a/runnables/test_suite_eval/pos_logs/1027.npy b/runnables/test_suite_eval/pos_logs/1027.npy
diff --git a/runnables/test_suite_eval/pos_logs/1028.npy b/runnables/test_suite_eval/pos_logs/1028.npy
diff --git a/runnables/test_suite_eval/pos_logs/1029.npy b/runnables/test_suite_eval/pos_logs/1029.npy
diff --git a/runnables/test_suite_eval/pos_logs/103.npy b/runnables/test_suite_eval/pos_logs/103.npy
diff --git a/runnables/test_suite_eval/pos_logs/1030.npy b/runnables/test_suite_eval/pos_logs/1030.npy
diff --git a/runnables/test_suite_eval/pos_logs/1031.npy b/runnables/test_suite_eval/pos_logs/1031.npy
diff --git a/runnables/test_suite_eval/pos_logs/1032.npy b/runnables/test_suite_eval/pos_logs/1032.npy
diff --git a/runnables/test_suite_eval/pos_logs/1033.npy b/runnables/test_suite_eval/pos_logs/1033.npy
diff --git a/runnables/test_suite_eval/pos_logs/1034.npy b/runnables/test_suite_eval/pos_logs/1034.npy
diff --git a/runnables/test_suite_eval/pos_logs/1035.npy b/runnables/test_suite_eval/pos_logs/1035.npy
diff --git a/runnables/test_suite_eval/pos_logs/1036.npy b/runnables/test_suite_eval/pos_logs/1036.npy
diff --git a/runnables/test_suite_eval/pos_logs/1037.npy b/runnables/test_suite_eval/pos_logs/1037.npy
diff --git a/runnables/test_suite_eval/pos_logs/1038.npy b/runnables/test_suite_eval/pos_logs/1038.npy
diff --git a/runnables/test_suite_eval/pos_logs/1039.npy b/runnables/test_suite_eval/pos_logs/1039.npy
diff --git a/runnables/test_suite_eval/pos_logs/104.npy b/runnables/test_suite_eval/pos_logs/104.npy
diff --git a/runnables/test_suite_eval/pos_logs/1040.npy b/runnables/test_suite_eval/pos_logs/1040.npy
diff --git a/runnables/test_suite_eval/pos_logs/1041.npy b/runnables/test_suite_eval/pos_logs/1041.npy
diff --git a/runnables/test_suite_eval/pos_logs/1042.npy b/runnables/test_suite_eval/pos_logs/1042.npy
diff --git a/runnables/test_suite_eval/pos_logs/1043.npy b/runnables/test_suite_eval/pos_logs/1043.npy
diff --git a/runnables/test_suite_eval/pos_logs/1044.npy b/runnables/test_suite_eval/pos_logs/1044.npy
diff --git a/runnables/test_suite_eval/pos_logs/1045.npy b/runnables/test_suite_eval/pos_logs/1045.npy
diff --git a/runnables/test_suite_eval/pos_logs/1046.npy b/runnables/test_suite_eval/pos_logs/1046.npy
diff --git a/runnables/test_suite_eval/pos_logs/1047.npy b/runnables/test_suite_eval/pos_logs/1047.npy
diff --git a/runnables/test_suite_eval/pos_logs/1048.npy b/runnables/test_suite_eval/pos_logs/1048.npy
diff --git a/runnables/test_suite_eval/pos_logs/1049.npy b/runnables/test_suite_eval/pos_logs/1049.npy
diff --git a/runnables/test_suite_eval/pos_logs/105.npy b/runnables/test_suite_eval/pos_logs/105.npy
diff --git a/runnables/test_suite_eval/pos_logs/1050.npy b/runnables/test_suite_eval/pos_logs/1050.npy
diff --git a/runnables/test_suite_eval/pos_logs/1051.npy b/runnables/test_suite_eval/pos_logs/1051.npy
diff --git a/runnables/test_suite_eval/pos_logs/1052.npy b/runnables/test_suite_eval/pos_logs/1052.npy
diff --git a/runnables/test_suite_eval/pos_logs/1053.npy b/runnables/test_suite_eval/pos_logs/1053.npy
diff --git a/runnables/test_suite_eval/pos_logs/1054.npy b/runnables/test_suite_eval/pos_logs/1054.npy
diff --git a/runnables/test_suite_eval/pos_logs/1055.npy b/runnables/test_suite_eval/pos_logs/1055.npy
diff --git a/runnables/test_suite_eval/pos_logs/1056.npy b/runnables/test_suite_eval/pos_logs/1056.npy
diff --git a/runnables/test_suite_eval/pos_logs/1057.npy b/runnables/test_suite_eval/pos_logs/1057.npy
diff --git a/runnables/test_suite_eval/pos_logs/1058.npy b/runnables/test_suite_eval/pos_logs/1058.npy
diff --git a/runnables/test_suite_eval/pos_logs/1059.npy b/runnables/test_suite_eval/pos_logs/1059.npy
diff --git a/runnables/test_suite_eval/pos_logs/106.npy b/runnables/test_suite_eval/pos_logs/106.npy
diff --git a/runnables/test_suite_eval/pos_logs/1060.npy b/runnables/test_suite_eval/pos_logs/1060.npy
diff --git a/runnables/test_suite_eval/pos_logs/1061.npy b/runnables/test_suite_eval/pos_logs/1061.npy
diff --git a/runnables/test_suite_eval/pos_logs/1062.npy b/runnables/test_suite_eval/pos_logs/1062.npy
diff --git a/runnables/test_suite_eval/pos_logs/1063.npy b/runnables/test_suite_eval/pos_logs/1063.npy
diff --git a/runnables/test_suite_eval/pos_logs/1064.npy b/runnables/test_suite_eval/pos_logs/1064.npy
diff --git a/runnables/test_suite_eval/pos_logs/1065.npy b/runnables/test_suite_eval/pos_logs/1065.npy
diff --git a/runnables/test_suite_eval/pos_logs/1066.npy b/runnables/test_suite_eval/pos_logs/1066.npy
diff --git a/runnables/test_suite_eval/pos_logs/1067.npy b/runnables/test_suite_eval/pos_logs/1067.npy
diff --git a/runnables/test_suite_eval/pos_logs/1068.npy b/runnables/test_suite_eval/pos_logs/1068.npy
diff --git a/runnables/test_suite_eval/pos_logs/1069.npy b/runnables/test_suite_eval/pos_logs/1069.npy
diff --git a/runnables/test_suite_eval/pos_logs/107.npy b/runnables/test_suite_eval/pos_logs/107.npy
diff --git a/runnables/test_suite_eval/pos_logs/1070.npy b/runnables/test_suite_eval/pos_logs/1070.npy
diff --git a/runnables/test_suite_eval/pos_logs/1071.npy b/runnables/test_suite_eval/pos_logs/1071.npy
diff --git a/runnables/test_suite_eval/pos_logs/1072.npy b/runnables/test_suite_eval/pos_logs/1072.npy
diff --git a/runnables/test_suite_eval/pos_logs/1073.npy b/runnables/test_suite_eval/pos_logs/1073.npy
diff --git a/runnables/test_suite_eval/pos_logs/1074.npy b/runnables/test_suite_eval/pos_logs/1074.npy
diff --git a/runnables/test_suite_eval/pos_logs/1075.npy b/runnables/test_suite_eval/pos_logs/1075.npy
diff --git a/runnables/test_suite_eval/pos_logs/1076.npy b/runnables/test_suite_eval/pos_logs/1076.npy
diff --git a/runnables/test_suite_eval/pos_logs/1077.npy b/runnables/test_suite_eval/pos_logs/1077.npy
diff --git a/runnables/test_suite_eval/pos_logs/1078.npy b/runnables/test_suite_eval/pos_logs/1078.npy
diff --git a/runnables/test_suite_eval/pos_logs/1079.npy b/runnables/test_suite_eval/pos_logs/1079.npy
diff --git a/runnables/test_suite_eval/pos_logs/108.npy b/runnables/test_suite_eval/pos_logs/108.npy
diff --git a/runnables/test_suite_eval/pos_logs/1080.npy b/runnables/test_suite_eval/pos_logs/1080.npy
diff --git a/runnables/test_suite_eval/pos_logs/1081.npy b/runnables/test_suite_eval/pos_logs/1081.npy
diff --git a/runnables/test_suite_eval/pos_logs/1082.npy b/runnables/test_suite_eval/pos_logs/1082.npy
diff --git a/runnables/test_suite_eval/pos_logs/1083.npy b/runnables/test_suite_eval/pos_logs/1083.npy
diff --git a/runnables/test_suite_eval/pos_logs/1084.npy b/runnables/test_suite_eval/pos_logs/1084.npy
diff --git a/runnables/test_suite_eval/pos_logs/1085.npy b/runnables/test_suite_eval/pos_logs/1085.npy
diff --git a/runnables/test_suite_eval/pos_logs/1086.npy b/runnables/test_suite_eval/pos_logs/1086.npy
diff --git a/runnables/test_suite_eval/pos_logs/1087.npy b/runnables/test_suite_eval/pos_logs/1087.npy
diff --git a/runnables/test_suite_eval/pos_logs/1088.npy b/runnables/test_suite_eval/pos_logs/1088.npy
diff --git a/runnables/test_suite_eval/pos_logs/1089.npy b/runnables/test_suite_eval/pos_logs/1089.npy
diff --git a/runnables/test_suite_eval/pos_logs/109.npy b/runnables/test_suite_eval/pos_logs/109.npy
diff --git a/runnables/test_suite_eval/pos_logs/1090.npy b/runnables/test_suite_eval/pos_logs/1090.npy
diff --git a/runnables/test_suite_eval/pos_logs/1091.npy b/runnables/test_suite_eval/pos_logs/1091.npy
diff --git a/runnables/test_suite_eval/pos_logs/1092.npy b/runnables/test_suite_eval/pos_logs/1092.npy
diff --git a/runnables/test_suite_eval/pos_logs/1093.npy b/runnables/test_suite_eval/pos_logs/1093.npy
diff --git a/runnables/test_suite_eval/pos_logs/1094.npy b/runnables/test_suite_eval/pos_logs/1094.npy
diff --git a/runnables/test_suite_eval/pos_logs/1095.npy b/runnables/test_suite_eval/pos_logs/1095.npy
diff --git a/runnables/test_suite_eval/pos_logs/1096.npy b/runnables/test_suite_eval/pos_logs/1096.npy
diff --git a/runnables/test_suite_eval/pos_logs/1097.npy b/runnables/test_suite_eval/pos_logs/1097.npy
diff --git a/runnables/test_suite_eval/pos_logs/1098.npy b/runnables/test_suite_eval/pos_logs/1098.npy
diff --git a/runnables/test_suite_eval/pos_logs/1099.npy b/runnables/test_suite_eval/pos_logs/1099.npy
diff --git a/runnables/test_suite_eval/pos_logs/11.npy b/runnables/test_suite_eval/pos_logs/11.npy
diff --git a/runnables/test_suite_eval/pos_logs/110.npy b/runnables/test_suite_eval/pos_logs/110.npy
diff --git a/runnables/test_suite_eval/pos_logs/1100.npy b/runnables/test_suite_eval/pos_logs/1100.npy
diff --git a/runnables/test_suite_eval/pos_logs/1101.npy b/runnables/test_suite_eval/pos_logs/1101.npy
diff --git a/runnables/test_suite_eval/pos_logs/1102.npy b/runnables/test_suite_eval/pos_logs/1102.npy
diff --git a/runnables/test_suite_eval/pos_logs/1103.npy b/runnables/test_suite_eval/pos_logs/1103.npy
diff --git a/runnables/test_suite_eval/pos_logs/1104.npy b/runnables/test_suite_eval/pos_logs/1104.npy
diff --git a/runnables/test_suite_eval/pos_logs/1105.npy b/runnables/test_suite_eval/pos_logs/1105.npy
diff --git a/runnables/test_suite_eval/pos_logs/1106.npy b/runnables/test_suite_eval/pos_logs/1106.npy
diff --git a/runnables/test_suite_eval/pos_logs/1107.npy b/runnables/test_suite_eval/pos_logs/1107.npy
diff --git a/runnables/test_suite_eval/pos_logs/1108.npy b/runnables/test_suite_eval/pos_logs/1108.npy
diff --git a/runnables/test_suite_eval/pos_logs/1109.npy b/runnables/test_suite_eval/pos_logs/1109.npy
diff --git a/runnables/test_suite_eval/pos_logs/111.npy b/runnables/test_suite_eval/pos_logs/111.npy
diff --git a/runnables/test_suite_eval/pos_logs/1110.npy b/runnables/test_suite_eval/pos_logs/1110.npy
diff --git a/runnables/test_suite_eval/pos_logs/1111.npy b/runnables/test_suite_eval/pos_logs/1111.npy
diff --git a/runnables/test_suite_eval/pos_logs/1112.npy b/runnables/test_suite_eval/pos_logs/1112.npy
diff --git a/runnables/test_suite_eval/pos_logs/1113.npy b/runnables/test_suite_eval/pos_logs/1113.npy
diff --git a/runnables/test_suite_eval/pos_logs/1114.npy b/runnables/test_suite_eval/pos_logs/1114.npy
diff --git a/runnables/test_suite_eval/pos_logs/1115.npy b/runnables/test_suite_eval/pos_logs/1115.npy
diff --git a/runnables/test_suite_eval/pos_logs/1116.npy b/runnables/test_suite_eval/pos_logs/1116.npy
diff --git a/runnables/test_suite_eval/pos_logs/1117.npy b/runnables/test_suite_eval/pos_logs/1117.npy
diff --git a/runnables/test_suite_eval/pos_logs/1118.npy b/runnables/test_suite_eval/pos_logs/1118.npy
diff --git a/runnables/test_suite_eval/pos_logs/1119.npy b/runnables/test_suite_eval/pos_logs/1119.npy
diff --git a/runnables/test_suite_eval/pos_logs/112.npy b/runnables/test_suite_eval/pos_logs/112.npy
diff --git a/runnables/test_suite_eval/pos_logs/1120.npy b/runnables/test_suite_eval/pos_logs/1120.npy
diff --git a/runnables/test_suite_eval/pos_logs/1121.npy b/runnables/test_suite_eval/pos_logs/1121.npy
diff --git a/runnables/test_suite_eval/pos_logs/1122.npy b/runnables/test_suite_eval/pos_logs/1122.npy
diff --git a/runnables/test_suite_eval/pos_logs/1123.npy b/runnables/test_suite_eval/pos_logs/1123.npy
diff --git a/runnables/test_suite_eval/pos_logs/1124.npy b/runnables/test_suite_eval/pos_logs/1124.npy
diff --git a/runnables/test_suite_eval/pos_logs/1125.npy b/runnables/test_suite_eval/pos_logs/1125.npy
diff --git a/runnables/test_suite_eval/pos_logs/1126.npy b/runnables/test_suite_eval/pos_logs/1126.npy
diff --git a/runnables/test_suite_eval/pos_logs/1127.npy b/runnables/test_suite_eval/pos_logs/1127.npy
diff --git a/runnables/test_suite_eval/pos_logs/1128.npy b/runnables/test_suite_eval/pos_logs/1128.npy
diff --git a/runnables/test_suite_eval/pos_logs/1129.npy b/runnables/test_suite_eval/pos_logs/1129.npy
diff --git a/runnables/test_suite_eval/pos_logs/113.npy b/runnables/test_suite_eval/pos_logs/113.npy
diff --git a/runnables/test_suite_eval/pos_logs/1130.npy b/runnables/test_suite_eval/pos_logs/1130.npy
diff --git a/runnables/test_suite_eval/pos_logs/1131.npy b/runnables/test_suite_eval/pos_logs/1131.npy
diff --git a/runnables/test_suite_eval/pos_logs/1132.npy b/runnables/test_suite_eval/pos_logs/1132.npy
diff --git a/runnables/test_suite_eval/pos_logs/1133.npy b/runnables/test_suite_eval/pos_logs/1133.npy
diff --git a/runnables/test_suite_eval/pos_logs/1134.npy b/runnables/test_suite_eval/pos_logs/1134.npy
diff --git a/runnables/test_suite_eval/pos_logs/1135.npy b/runnables/test_suite_eval/pos_logs/1135.npy
diff --git a/runnables/test_suite_eval/pos_logs/1136.npy b/runnables/test_suite_eval/pos_logs/1136.npy
diff --git a/runnables/test_suite_eval/pos_logs/1137.npy b/runnables/test_suite_eval/pos_logs/1137.npy
diff --git a/runnables/test_suite_eval/pos_logs/1138.npy b/runnables/test_suite_eval/pos_logs/1138.npy
diff --git a/runnables/test_suite_eval/pos_logs/1139.npy b/runnables/test_suite_eval/pos_logs/1139.npy
diff --git a/runnables/test_suite_eval/pos_logs/114.npy b/runnables/test_suite_eval/pos_logs/114.npy
diff --git a/runnables/test_suite_eval/pos_logs/1140.npy b/runnables/test_suite_eval/pos_logs/1140.npy
diff --git a/runnables/test_suite_eval/pos_logs/1141.npy b/runnables/test_suite_eval/pos_logs/1141.npy
diff --git a/runnables/test_suite_eval/pos_logs/1142.npy b/runnables/test_suite_eval/pos_logs/1142.npy
diff --git a/runnables/test_suite_eval/pos_logs/1143.npy b/runnables/test_suite_eval/pos_logs/1143.npy
diff --git a/runnables/test_suite_eval/pos_logs/1144.npy b/runnables/test_suite_eval/pos_logs/1144.npy
diff --git a/runnables/test_suite_eval/pos_logs/1145.npy b/runnables/test_suite_eval/pos_logs/1145.npy
diff --git a/runnables/test_suite_eval/pos_logs/1146.npy b/runnables/test_suite_eval/pos_logs/1146.npy
diff --git a/runnables/test_suite_eval/pos_logs/1147.npy b/runnables/test_suite_eval/pos_logs/1147.npy
diff --git a/runnables/test_suite_eval/pos_logs/1148.npy b/runnables/test_suite_eval/pos_logs/1148.npy
diff --git a/runnables/test_suite_eval/pos_logs/1149.npy b/runnables/test_suite_eval/pos_logs/1149.npy
diff --git a/runnables/test_suite_eval/pos_logs/115.npy b/runnables/test_suite_eval/pos_logs/115.npy
diff --git a/runnables/test_suite_eval/pos_logs/1150.npy b/runnables/test_suite_eval/pos_logs/1150.npy
diff --git a/runnables/test_suite_eval/pos_logs/1151.npy b/runnables/test_suite_eval/pos_logs/1151.npy
diff --git a/runnables/test_suite_eval/pos_logs/1152.npy b/runnables/test_suite_eval/pos_logs/1152.npy
diff --git a/runnables/test_suite_eval/pos_logs/1153.npy b/runnables/test_suite_eval/pos_logs/1153.npy
diff --git a/runnables/test_suite_eval/pos_logs/1154.npy b/runnables/test_suite_eval/pos_logs/1154.npy
diff --git a/runnables/test_suite_eval/pos_logs/1155.npy b/runnables/test_suite_eval/pos_logs/1155.npy
diff --git a/runnables/test_suite_eval/pos_logs/1156.npy b/runnables/test_suite_eval/pos_logs/1156.npy
diff --git a/runnables/test_suite_eval/pos_logs/1157.npy b/runnables/test_suite_eval/pos_logs/1157.npy
diff --git a/runnables/test_suite_eval/pos_logs/1158.npy b/runnables/test_suite_eval/pos_logs/1158.npy
diff --git a/runnables/test_suite_eval/pos_logs/1159.npy b/runnables/test_suite_eval/pos_logs/1159.npy
diff --git a/runnables/test_suite_eval/pos_logs/116.npy b/runnables/test_suite_eval/pos_logs/116.npy
diff --git a/runnables/test_suite_eval/pos_logs/1160.npy b/runnables/test_suite_eval/pos_logs/1160.npy
diff --git a/runnables/test_suite_eval/pos_logs/1161.npy b/runnables/test_suite_eval/pos_logs/1161.npy
diff --git a/runnables/test_suite_eval/pos_logs/1162.npy b/runnables/test_suite_eval/pos_logs/1162.npy
diff --git a/runnables/test_suite_eval/pos_logs/1163.npy b/runnables/test_suite_eval/pos_logs/1163.npy
diff --git a/runnables/test_suite_eval/pos_logs/1164.npy b/runnables/test_suite_eval/pos_logs/1164.npy
diff --git a/runnables/test_suite_eval/pos_logs/1165.npy b/runnables/test_suite_eval/pos_logs/1165.npy
diff --git a/runnables/test_suite_eval/pos_logs/1166.npy b/runnables/test_suite_eval/pos_logs/1166.npy
diff --git a/runnables/test_suite_eval/pos_logs/1167.npy b/runnables/test_suite_eval/pos_logs/1167.npy
diff --git a/runnables/test_suite_eval/pos_logs/1168.npy b/runnables/test_suite_eval/pos_logs/1168.npy
diff --git a/runnables/test_suite_eval/pos_logs/1169.npy b/runnables/test_suite_eval/pos_logs/1169.npy
diff --git a/runnables/test_suite_eval/pos_logs/117.npy b/runnables/test_suite_eval/pos_logs/117.npy
diff --git a/runnables/test_suite_eval/pos_logs/1170.npy b/runnables/test_suite_eval/pos_logs/1170.npy
diff --git a/runnables/test_suite_eval/pos_logs/1171.npy b/runnables/test_suite_eval/pos_logs/1171.npy
diff --git a/runnables/test_suite_eval/pos_logs/1172.npy b/runnables/test_suite_eval/pos_logs/1172.npy
diff --git a/runnables/test_suite_eval/pos_logs/1173.npy b/runnables/test_suite_eval/pos_logs/1173.npy
diff --git a/runnables/test_suite_eval/pos_logs/1174.npy b/runnables/test_suite_eval/pos_logs/1174.npy
diff --git a/runnables/test_suite_eval/pos_logs/1175.npy b/runnables/test_suite_eval/pos_logs/1175.npy
diff --git a/runnables/test_suite_eval/pos_logs/1176.npy b/runnables/test_suite_eval/pos_logs/1176.npy
diff --git a/runnables/test_suite_eval/pos_logs/1177.npy b/runnables/test_suite_eval/pos_logs/1177.npy
diff --git a/runnables/test_suite_eval/pos_logs/1178.npy b/runnables/test_suite_eval/pos_logs/1178.npy
diff --git a/runnables/test_suite_eval/pos_logs/1179.npy b/runnables/test_suite_eval/pos_logs/1179.npy
diff --git a/runnables/test_suite_eval/pos_logs/118.npy b/runnables/test_suite_eval/pos_logs/118.npy
diff --git a/runnables/test_suite_eval/pos_logs/1180.npy b/runnables/test_suite_eval/pos_logs/1180.npy
diff --git a/runnables/test_suite_eval/pos_logs/1181.npy b/runnables/test_suite_eval/pos_logs/1181.npy
diff --git a/runnables/test_suite_eval/pos_logs/1182.npy b/runnables/test_suite_eval/pos_logs/1182.npy
diff --git a/runnables/test_suite_eval/pos_logs/1183.npy b/runnables/test_suite_eval/pos_logs/1183.npy
diff --git a/runnables/test_suite_eval/pos_logs/1184.npy b/runnables/test_suite_eval/pos_logs/1184.npy
diff --git a/runnables/test_suite_eval/pos_logs/1185.npy b/runnables/test_suite_eval/pos_logs/1185.npy
diff --git a/runnables/test_suite_eval/pos_logs/1186.npy b/runnables/test_suite_eval/pos_logs/1186.npy
diff --git a/runnables/test_suite_eval/pos_logs/1187.npy b/runnables/test_suite_eval/pos_logs/1187.npy
diff --git a/runnables/test_suite_eval/pos_logs/1188.npy b/runnables/test_suite_eval/pos_logs/1188.npy
diff --git a/runnables/test_suite_eval/pos_logs/1189.npy b/runnables/test_suite_eval/pos_logs/1189.npy
diff --git a/runnables/test_suite_eval/pos_logs/119.npy b/runnables/test_suite_eval/pos_logs/119.npy
diff --git a/runnables/test_suite_eval/pos_logs/1190.npy b/runnables/test_suite_eval/pos_logs/1190.npy
diff --git a/runnables/test_suite_eval/pos_logs/1191.npy b/runnables/test_suite_eval/pos_logs/1191.npy
diff --git a/runnables/test_suite_eval/pos_logs/1192.npy b/runnables/test_suite_eval/pos_logs/1192.npy
diff --git a/runnables/test_suite_eval/pos_logs/1193.npy b/runnables/test_suite_eval/pos_logs/1193.npy
diff --git a/runnables/test_suite_eval/pos_logs/1194.npy b/runnables/test_suite_eval/pos_logs/1194.npy
diff --git a/runnables/test_suite_eval/pos_logs/1195.npy b/runnables/test_suite_eval/pos_logs/1195.npy
diff --git a/runnables/test_suite_eval/pos_logs/1196.npy b/runnables/test_suite_eval/pos_logs/1196.npy
diff --git a/runnables/test_suite_eval/pos_logs/1197.npy b/runnables/test_suite_eval/pos_logs/1197.npy
diff --git a/runnables/test_suite_eval/pos_logs/1198.npy b/runnables/test_suite_eval/pos_logs/1198.npy
diff --git a/runnables/test_suite_eval/pos_logs/1199.npy b/runnables/test_suite_eval/pos_logs/1199.npy
diff --git a/runnables/test_suite_eval/pos_logs/12.npy b/runnables/test_suite_eval/pos_logs/12.npy
diff --git a/runnables/test_suite_eval/pos_logs/120.npy b/runnables/test_suite_eval/pos_logs/120.npy
diff --git a/runnables/test_suite_eval/pos_logs/1200.npy b/runnables/test_suite_eval/pos_logs/1200.npy
diff --git a/runnables/test_suite_eval/pos_logs/1201.npy b/runnables/test_suite_eval/pos_logs/1201.npy
diff --git a/runnables/test_suite_eval/pos_logs/1202.npy b/runnables/test_suite_eval/pos_logs/1202.npy
diff --git a/runnables/test_suite_eval/pos_logs/1203.npy b/runnables/test_suite_eval/pos_logs/1203.npy
diff --git a/runnables/test_suite_eval/pos_logs/1204.npy b/runnables/test_suite_eval/pos_logs/1204.npy
diff --git a/runnables/test_suite_eval/pos_logs/1205.npy b/runnables/test_suite_eval/pos_logs/1205.npy
diff --git a/runnables/test_suite_eval/pos_logs/1206.npy b/runnables/test_suite_eval/pos_logs/1206.npy
diff --git a/runnables/test_suite_eval/pos_logs/1207.npy b/runnables/test_suite_eval/pos_logs/1207.npy
diff --git a/runnables/test_suite_eval/pos_logs/1208.npy b/runnables/test_suite_eval/pos_logs/1208.npy
diff --git a/runnables/test_suite_eval/pos_logs/1209.npy b/runnables/test_suite_eval/pos_logs/1209.npy
diff --git a/runnables/test_suite_eval/pos_logs/121.npy b/runnables/test_suite_eval/pos_logs/121.npy
diff --git a/runnables/test_suite_eval/pos_logs/1210.npy b/runnables/test_suite_eval/pos_logs/1210.npy
diff --git a/runnables/test_suite_eval/pos_logs/1211.npy b/runnables/test_suite_eval/pos_logs/1211.npy
diff --git a/runnables/test_suite_eval/pos_logs/1212.npy b/runnables/test_suite_eval/pos_logs/1212.npy
diff --git a/runnables/test_suite_eval/pos_logs/1213.npy b/runnables/test_suite_eval/pos_logs/1213.npy
diff --git a/runnables/test_suite_eval/pos_logs/1214.npy b/runnables/test_suite_eval/pos_logs/1214.npy
diff --git a/runnables/test_suite_eval/pos_logs/1215.npy b/runnables/test_suite_eval/pos_logs/1215.npy
diff --git a/runnables/test_suite_eval/pos_logs/1216.npy b/runnables/test_suite_eval/pos_logs/1216.npy
diff --git a/runnables/test_suite_eval/pos_logs/1217.npy b/runnables/test_suite_eval/pos_logs/1217.npy
diff --git a/runnables/test_suite_eval/pos_logs/1218.npy b/runnables/test_suite_eval/pos_logs/1218.npy
diff --git a/runnables/test_suite_eval/pos_logs/1219.npy b/runnables/test_suite_eval/pos_logs/1219.npy
diff --git a/runnables/test_suite_eval/pos_logs/122.npy b/runnables/test_suite_eval/pos_logs/122.npy
diff --git a/runnables/test_suite_eval/pos_logs/1220.npy b/runnables/test_suite_eval/pos_logs/1220.npy
diff --git a/runnables/test_suite_eval/pos_logs/1221.npy b/runnables/test_suite_eval/pos_logs/1221.npy
diff --git a/runnables/test_suite_eval/pos_logs/1222.npy b/runnables/test_suite_eval/pos_logs/1222.npy
diff --git a/runnables/test_suite_eval/pos_logs/1223.npy b/runnables/test_suite_eval/pos_logs/1223.npy
diff --git a/runnables/test_suite_eval/pos_logs/1224.npy b/runnables/test_suite_eval/pos_logs/1224.npy
diff --git a/runnables/test_suite_eval/pos_logs/1225.npy b/runnables/test_suite_eval/pos_logs/1225.npy
diff --git a/runnables/test_suite_eval/pos_logs/1226.npy b/runnables/test_suite_eval/pos_logs/1226.npy
diff --git a/runnables/test_suite_eval/pos_logs/1227.npy b/runnables/test_suite_eval/pos_logs/1227.npy
diff --git a/runnables/test_suite_eval/pos_logs/1228.npy b/runnables/test_suite_eval/pos_logs/1228.npy
diff --git a/runnables/test_suite_eval/pos_logs/1229.npy b/runnables/test_suite_eval/pos_logs/1229.npy
diff --git a/runnables/test_suite_eval/pos_logs/123.npy b/runnables/test_suite_eval/pos_logs/123.npy
diff --git a/runnables/test_suite_eval/pos_logs/1230.npy b/runnables/test_suite_eval/pos_logs/1230.npy
diff --git a/runnables/test_suite_eval/pos_logs/1231.npy b/runnables/test_suite_eval/pos_logs/1231.npy
diff --git a/runnables/test_suite_eval/pos_logs/1232.npy b/runnables/test_suite_eval/pos_logs/1232.npy
diff --git a/runnables/test_suite_eval/pos_logs/1233.npy b/runnables/test_suite_eval/pos_logs/1233.npy
diff --git a/runnables/test_suite_eval/pos_logs/1234.npy b/runnables/test_suite_eval/pos_logs/1234.npy
diff --git a/runnables/test_suite_eval/pos_logs/1235.npy b/runnables/test_suite_eval/pos_logs/1235.npy
diff --git a/runnables/test_suite_eval/pos_logs/1236.npy b/runnables/test_suite_eval/pos_logs/1236.npy
diff --git a/runnables/test_suite_eval/pos_logs/1237.npy b/runnables/test_suite_eval/pos_logs/1237.npy
diff --git a/runnables/test_suite_eval/pos_logs/1238.npy b/runnables/test_suite_eval/pos_logs/1238.npy
diff --git a/runnables/test_suite_eval/pos_logs/1239.npy b/runnables/test_suite_eval/pos_logs/1239.npy
diff --git a/runnables/test_suite_eval/pos_logs/124.npy b/runnables/test_suite_eval/pos_logs/124.npy
diff --git a/runnables/test_suite_eval/pos_logs/1240.npy b/runnables/test_suite_eval/pos_logs/1240.npy
diff --git a/runnables/test_suite_eval/pos_logs/1241.npy b/runnables/test_suite_eval/pos_logs/1241.npy
diff --git a/runnables/test_suite_eval/pos_logs/1242.npy b/runnables/test_suite_eval/pos_logs/1242.npy
diff --git a/runnables/test_suite_eval/pos_logs/1243.npy b/runnables/test_suite_eval/pos_logs/1243.npy
diff --git a/runnables/test_suite_eval/pos_logs/1244.npy b/runnables/test_suite_eval/pos_logs/1244.npy
diff --git a/runnables/test_suite_eval/pos_logs/1245.npy b/runnables/test_suite_eval/pos_logs/1245.npy
diff --git a/runnables/test_suite_eval/pos_logs/1246.npy b/runnables/test_suite_eval/pos_logs/1246.npy
diff --git a/runnables/test_suite_eval/pos_logs/1247.npy b/runnables/test_suite_eval/pos_logs/1247.npy
diff --git a/runnables/test_suite_eval/pos_logs/1248.npy b/runnables/test_suite_eval/pos_logs/1248.npy
diff --git a/runnables/test_suite_eval/pos_logs/1249.npy b/runnables/test_suite_eval/pos_logs/1249.npy
diff --git a/runnables/test_suite_eval/pos_logs/125.npy b/runnables/test_suite_eval/pos_logs/125.npy
diff --git a/runnables/test_suite_eval/pos_logs/1250.npy b/runnables/test_suite_eval/pos_logs/1250.npy
diff --git a/runnables/test_suite_eval/pos_logs/1251.npy b/runnables/test_suite_eval/pos_logs/1251.npy
diff --git a/runnables/test_suite_eval/pos_logs/1252.npy b/runnables/test_suite_eval/pos_logs/1252.npy
diff --git a/runnables/test_suite_eval/pos_logs/1253.npy b/runnables/test_suite_eval/pos_logs/1253.npy
diff --git a/runnables/test_suite_eval/pos_logs/1254.npy b/runnables/test_suite_eval/pos_logs/1254.npy
diff --git a/runnables/test_suite_eval/pos_logs/1255.npy b/runnables/test_suite_eval/pos_logs/1255.npy
diff --git a/runnables/test_suite_eval/pos_logs/1256.npy b/runnables/test_suite_eval/pos_logs/1256.npy
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{"success_rate": 0.24812030075187969, "avg mean dev": 0.12312815303888709, "avg max dev": 0.46306573291494524, "avt time": 25.69886363636364}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{"success_rate": 0.36507936507936506, "avg mean dev": 0.10840245515779884, "avg max dev": 0.36533508928277664, "avt time": 26.56811594202898}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{"success_rate": 0.4791666666666667, "avg mean dev": 0.09615903659247474, "avg max dev": 0.28994711420539, "avt time": 26.77921195652174}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{"success_rate": 0.49748743718592964, "avg mean dev": 0.08853108014915095, "avg max dev": 0.22978509050899154, "avt time": 27.1082702020202}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{"success_rate": 0.5, "avg mean dev": 0.08308981149934497, "avg max dev": 0.20903981543228453, "avt time": 28.1221875}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{"success_rate": 0.985, "avg mean dev": 0.1772167915118993, "avg max dev": 0.4917212704683174, "avt time": 14.34890016920474}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{"success_rate": 0.99, "avg mean dev": 0.1433386324227451, "avg max dev": 0.392061174557603, "avt time": 16.289898989898987}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{"success_rate": 0.995, "avg mean dev": 0.10246450706590106, "avg max dev": 0.27034303375162694, "avt time": 20.263651591289783}