-
Notifications
You must be signed in to change notification settings - Fork 1
/
benchmark_EDA.py
173 lines (137 loc) · 7.17 KB
/
benchmark_EDA.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
from matplotlib import tight_layout
from benchmark_handler import BenchmarkHandler
import itertools
import pandas as pd
import numpy as np
import seaborn as sns
from matplotlib import pyplot as plt
from sklearn import feature_selection as f
sns.set() # Setting seaborn as default style even if use only matplotlib
def get_aggregated_results (benchmark, space_name="PPO", budget= 99, metric = "eval_avg_returns", smoke_test = False):
search_space = benchmark.get_search_space(space_name)
hps_names = list(search_space.keys())
environments = benchmark.get_environments()
if smoke_test:
environments = environments[:3]
ranks = []
all_results = []
failed_confs = []
for seed in [0,1,2]:
temp_seed_ranks = []
for environment in environments:
results_history = []
try:
benchmark.set_env_space_seed(search_space=space_name, environment=environment, seed=seed)
temp_failed_confs = []
for hps in itertools.product(*tuple(list(search_space.values()))):
try:
configuration = dict(zip(hps_names, hps))
query = benchmark.get_metrics(configuration, budget=budget)[metric]
configuration["response"] = query[-1]
configuration[metric] = metric
configuration["environment"] = environment
configuration["conf_id"] = str(hps)
results_history.append(configuration)
except Exception as e:
print(e)
temp_failed_confs.append(hps)
print(f"Problem at config: {configuration}")
results_df = pd.DataFrame(results_history)
rank = results_df["response"].rank(ascending=False).values.tolist()
temp_seed_ranks.append(rank)
all_results.append(results_df)
failed_confs.extend(temp_failed_confs)
all_configurations = results_df["conf_id"]
except Exception as e:
#print("env:", environment, " space:", space_name)
print(e)
pass
ranks.append(temp_seed_ranks)
#failed_confs = list(set(failed_confs))
#all_configurations = list(itertools.product(*tuple(search_space.values())))
#all_configurations = [str(x) for x in all_configurations if x not in failed_confs ]
aggregated_results = pd.DataFrame()
for rank in ranks:
temp_df = pd.DataFrame(rank)
temp_df = temp_df.T.dropna().T
temp_df.columns = all_configurations
aggregated_results = pd.concat((aggregated_results, temp_df), axis=0)
features_rank = []
for result in all_results:
temp_rank=pd.DataFrame(f.f_regression(result[hps_names], result["response"])[0]).rank(ascending=False).values.reshape(-1).tolist()
#temp_rank=pd.DataFrame(f.f_regression(result[hps_names], result["response"])[0]).values.reshape(-1).tolist()
features_rank.append(temp_rank)
return aggregated_results, features_rank, all_configurations, hps_names
def plot_catplot_on_axis(benchmark, space_name, smoke_test = False):
aggregated_results, features_rank, all_configurations, hps = get_aggregated_results(benchmark, space_name, smoke_test = smoke_test)
mean_aggregated_results = aggregated_results.mean(axis=0)
ix_sort = mean_aggregated_results.argsort()
best = ix_sort[:n_configurations_to_plot].tolist()
worst = ix_sort[-n_configurations_to_plot:].tolist()
selected_configurations = []
for config_id in best+worst:
selected_configurations.append(all_configurations[config_id])
subset_results = pd.DataFrame(aggregated_results.iloc[:, best+worst])
subset_results.columns = selected_configurations
configuration_structure = str(tuple (benchmark.get_search_space(space_name).keys()))
return subset_results, features_rank, configuration_structure, hps
benchmark = BenchmarkHandler()
n_configurations_to_plot = 4
linewidth = 2
smoke_test = False
fontsize=70
boxprops = dict(linestyle='--', linewidth=linewidth)
flierprops = dict(marker='o', markerfacecolor='green', markersize=12,
markeredgecolor='none')
medianprops = dict(linestyle='-', linewidth=linewidth*3, color='firebrick')
meanpointprops = dict(marker='D', markeredgecolor='black', markersize=12, markerfacecolor='firebrick')
meanlineprops = dict(linestyle='--', linewidth=linewidth, color='purple')
algorithms = ["PPO","A2C", "DDPG", "SAC", "TD3", "DQN"]
n_algorithms = len(algorithms)
fig1, axis1 = plt.subplots(1,n_algorithms,figsize=(15*n_algorithms,15))
fig2, axis2 = plt.subplots(1,n_algorithms,figsize=(15*n_algorithms,15))
plt.rcParams.update({
"font.family": "serif",
})
for i, space_name in enumerate(algorithms):
subset_results, features_rank, configuration_structure, hps = plot_catplot_on_axis(benchmark, space_name, smoke_test)
selected_configurations = subset_results.columns
bplot1 = axis1[i].boxplot(subset_results, patch_artist=True, boxprops=boxprops,
flierprops=flierprops,
meanprops=meanpointprops,
medianprops=medianprops)
features_rank = np.array(features_rank)
features_rank[np.isnan(features_rank)] = 1
bplot2 = axis2[i].boxplot(features_rank, patch_artist=True, boxprops=boxprops,
flierprops=flierprops,
# meanpointprops=meanpointprops,
meanprops=meanpointprops,
medianprops=medianprops,
showmeans=True)
#axis2[i].violinplot(np.array(features_rank))
n = 8
cm = plt.cm.get_cmap("Paired")
colors = [cm(val/n) for val in range(n)]
for patch, color in zip(bplot1['boxes'], colors):
patch.set_facecolor(color)
axis1[i].set_xticklabels(selected_configurations, rotation=90, fontsize=fontsize)
axis1[i].yaxis.set_tick_params(labelsize=fontsize)
axis1[i].set_title(space_name, fontsize=fontsize)
axis1[i].set_xlabel(configuration_structure, fontsize=fontsize)
for patch, color in zip(bplot2['boxes'], colors):
patch.set_facecolor(color)
axis2[i].set_xticklabels(hps, rotation=0, fontsize=fontsize)
axis2[i].yaxis.set_tick_params(labelsize=fontsize)
axis2[i].set_title(space_name, fontsize=fontsize)
if i==0:
axis1[i].set_ylabel("Rank of Reward", fontsize=fontsize)
axis1[i].xaxis.set_label_coords(.5, -.8)
if i==0:
axis2[i].set_ylabel("Rank of Importance", fontsize=fontsize)
if i==2:
axis2[i].set_xlabel("Hyperparameters", fontsize=fontsize)
axis2[i].xaxis.set_label_coords(.5, -.2)
#plt.show()
plt.tight_layout()
fig1.savefig("plots/average_rank_per_algorithm.pdf", bbox_inches="tight")
fig2.savefig("plots/feature_importance_per_algorithm.pdf", bbox_inches="tight")