Skip to content

Commit df06ae3

Browse files
lvhan028lvhan028
authored
update pre-commit config (#2683)
* update pre-commit config * update * change --wrap-descriptions to 120 * update * update * update * isort * apply all * merge main * isort line-length 120 * upddate
1 parent 3f8b079 commit df06ae3

319 files changed

Lines changed: 6278 additions & 12936 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/scripts/action_tools.py‎

Lines changed: 18 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -37,11 +37,7 @@ def run_cmd(cmd_lines: List[str], log_path: str, cwd: str = None):
3737
with open(log_path, 'w', encoding='utf-8') as file_handler:
3838
file_handler.write(f'Command:\n{cmd_for_log}\n')
3939
file_handler.flush()
40-
process_res = subprocess.Popen(cmd_for_run,
41-
shell=True,
42-
cwd=cwd,
43-
stdout=file_handler,
44-
stderr=file_handler)
40+
process_res = subprocess.Popen(cmd_for_run, shell=True, cwd=cwd, stdout=file_handler, stderr=file_handler)
4541
process_res.wait()
4642
return_code = process_res.returncode
4743

@@ -79,11 +75,7 @@ def add_summary(csv_path: str):
7975
_append_summary('\n')
8076

8177

82-
def evaluate(models: List[str],
83-
datasets: List[str],
84-
workspace: str,
85-
evaluate_type: str,
86-
is_smoke: bool = False):
78+
def evaluate(models: List[str], datasets: List[str], workspace: str, evaluate_type: str, is_smoke: bool = False):
8779
"""Evaluate models from lmdeploy using opencompass.
8880
8981
Args:
@@ -103,10 +95,8 @@ def evaluate(models: List[str],
10395

10496
opencompass_dir = os.path.abspath(os.environ['OPENCOMPASS_DIR'])
10597
lmdeploy_dir = os.path.abspath(os.environ['LMDEPLOY_DIR'])
106-
config_path = os.path.join(
107-
lmdeploy_dir, f'.github/scripts/eval_{evaluate_type}_config.py')
108-
config_path_new = os.path.join(opencompass_dir, 'configs',
109-
'eval_lmdeploy.py')
98+
config_path = os.path.join(lmdeploy_dir, f'.github/scripts/eval_{evaluate_type}_config.py')
99+
config_path_new = os.path.join(opencompass_dir, 'configs', 'eval_lmdeploy.py')
110100
if os.path.exists(config_path_new):
111101
os.remove(config_path_new)
112102
shutil.copy(config_path, config_path_new)
@@ -164,9 +154,7 @@ def evaluate(models: List[str],
164154
row = [_.strip() for _ in row]
165155
if row[-1] != '-':
166156
model_results[row[0]] = row[-1]
167-
crows_pairs_json = glob.glob(os.path.join(
168-
work_dir, '*/results/*/crows_pairs.json'),
169-
recursive=True)
157+
crows_pairs_json = glob.glob(os.path.join(work_dir, '*/results/*/crows_pairs.json'), recursive=True)
170158
if len(crows_pairs_json) == 1:
171159
with open(crows_pairs_json[0], 'r') as f:
172160
acc = json.load(f)['accuracy']
@@ -175,13 +163,11 @@ def evaluate(models: List[str],
175163
logging.info(f'\n{model}\n{model_results}')
176164
dataset_names = list(model_results.keys())
177165

178-
row = ','.join([model, str(task_duration_seconds)] +
179-
[model_results[_] for _ in dataset_names])
166+
row = ','.join([model, str(task_duration_seconds)] + [model_results[_] for _ in dataset_names])
180167

181168
if not os.path.exists(output_csv):
182169
with open(output_csv, 'w') as f:
183-
header = ','.join(['Model', 'task_duration_secs'] +
184-
dataset_names)
170+
header = ','.join(['Model', 'task_duration_secs'] + dataset_names)
185171
f.write(header + '\n')
186172
f.write(row + '\n')
187173
else:
@@ -213,30 +199,22 @@ def generate_benchmark_report(report_path: str):
213199
_append_summary('## Benchmark Results Start')
214200
subfolders = [f.path for f in os.scandir(report_path) if f.is_dir()]
215201
for dir_path in subfolders:
216-
second_subfolders = [
217-
f.path for f in sorted(os.scandir(dir_path), key=lambda x: x.name)
218-
if f.is_dir()
219-
]
202+
second_subfolders = [f.path for f in sorted(os.scandir(dir_path), key=lambda x: x.name) if f.is_dir()]
220203
for sec_dir_path in second_subfolders:
221204
model = sec_dir_path.replace(report_path + '/', '')
222205
print('-' * 25, model, '-' * 25)
223206
_append_summary('-' * 25 + model + '-' * 25 + '\n')
224207

225208
benchmark_subfolders = [
226-
f.path
227-
for f in sorted(os.scandir(sec_dir_path), key=lambda x: x.name)
228-
if f.is_dir()
209+
f.path for f in sorted(os.scandir(sec_dir_path), key=lambda x: x.name) if f.is_dir()
229210
]
230211
for backend_subfolder in benchmark_subfolders:
231-
benchmark_type = backend_subfolder.replace(
232-
sec_dir_path + '/', '')
212+
benchmark_type = backend_subfolder.replace(sec_dir_path + '/', '')
233213
print('*' * 10, benchmark_type, '*' * 10)
234214
_append_summary('-' * 10 + benchmark_type + '-' * 10 + '\n')
235-
merged_csv_path = os.path.join(backend_subfolder,
236-
'summary.csv')
215+
merged_csv_path = os.path.join(backend_subfolder, 'summary.csv')
237216
csv_files = glob.glob(os.path.join(backend_subfolder, '*.csv'))
238-
average_csv_path = os.path.join(backend_subfolder,
239-
'average.csv')
217+
average_csv_path = os.path.join(backend_subfolder, 'average.csv')
240218
if merged_csv_path in csv_files:
241219
csv_files.remove(merged_csv_path)
242220
if average_csv_path in csv_files:
@@ -246,21 +224,18 @@ def generate_benchmark_report(report_path: str):
246224
if len(csv_files) > 0:
247225
for f in csv_files:
248226
df = pd.read_csv(f)
249-
merged_df = pd.concat([merged_df, df],
250-
ignore_index=True)
227+
merged_df = pd.concat([merged_df, df], ignore_index=True)
251228

252229
merged_df = merged_df.sort_values(by=merged_df.columns[0])
253230

254231
grouped_df = merged_df.groupby(merged_df.columns[0])
255232
if 'generation' not in backend_subfolder:
256-
average_values = grouped_df.pipe(
257-
(lambda group: {
258-
'mean': group.mean().round(decimals=3)
259-
}))['mean']
233+
average_values = grouped_df.pipe((lambda group: {
234+
'mean': group.mean().round(decimals=3)
235+
}))['mean']
260236
average_values.to_csv(average_csv_path, index=True)
261237
avg_df = pd.read_csv(average_csv_path)
262-
merged_df = pd.concat([merged_df, avg_df],
263-
ignore_index=True)
238+
merged_df = pd.concat([merged_df, avg_df], ignore_index=True)
264239
add_summary(average_csv_path)
265240
merged_df.to_csv(merged_csv_path, index=False)
266241
if 'generation' in backend_subfolder:
@@ -287,10 +262,7 @@ def generate_csv_from_profile_result(file_path: str, out_path: str):
287262
import csv
288263
with open(out_path, 'w', newline='') as f:
289264
writer = csv.writer(f)
290-
writer.writerow([
291-
'request_rate', 'completed', 'RPM', 'median_ttft_ms',
292-
'output_throughput'
293-
])
265+
writer.writerow(['request_rate', 'completed', 'RPM', 'median_ttft_ms', 'output_throughput'])
294266
writer.writerows(data_csv)
295267

296268

‎.github/scripts/doc_link_checker.py‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,8 @@
88

99
def make_parser():
1010
parser = argparse.ArgumentParser('Doc link checker')
11-
parser.add_argument('--http',
12-
default=False,
13-
type=bool,
14-
help='check http or not ')
15-
parser.add_argument('--target',
16-
default='./docs',
17-
type=str,
18-
help='the directory or file to check')
11+
parser.add_argument('--http', default=False, type=bool, help='check http or not ')
12+
parser.add_argument('--target', default='./docs', type=str, help='the directory or file to check')
1913
return parser
2014

2115

‎.github/scripts/eval_base_config.py‎

Lines changed: 36 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -4,102 +4,67 @@
44

55
with read_base():
66
# choose a list of datasets
7-
from opencompass.configs.datasets.ARC_c.ARC_c_few_shot_ppl import \
8-
ARC_c_datasets # noqa: F401, E501
9-
from opencompass.configs.datasets.bbh.bbh_gen_98fba6 import \
10-
bbh_datasets # noqa: F401, E501
11-
from opencompass.configs.datasets.ceval.ceval_ppl import \
12-
ceval_datasets # noqa: F401, E501
13-
from opencompass.configs.datasets.cmmlu.cmmlu_ppl_041cbf import \
14-
cmmlu_datasets # noqa: F401, E501
15-
from opencompass.configs.datasets.crowspairs.crowspairs_ppl import \
16-
crowspairs_datasets # noqa: F401, E501
17-
from opencompass.configs.datasets.drop.drop_gen_a2697c import \
18-
drop_datasets # noqa: F401, E501
7+
from opencompass.configs.datasets.ARC_c.ARC_c_few_shot_ppl import ARC_c_datasets # noqa: F401, E501
8+
from opencompass.configs.datasets.bbh.bbh_gen_98fba6 import bbh_datasets # noqa: F401, E501
9+
from opencompass.configs.datasets.ceval.ceval_ppl import ceval_datasets # noqa: F401, E501
10+
from opencompass.configs.datasets.cmmlu.cmmlu_ppl_041cbf import cmmlu_datasets # noqa: F401, E501
11+
from opencompass.configs.datasets.crowspairs.crowspairs_ppl import crowspairs_datasets # noqa: F401, E501
12+
from opencompass.configs.datasets.drop.drop_gen_a2697c import drop_datasets # noqa: F401, E501
1913
# Corebench v1.7
2014
from opencompass.configs.datasets.GaokaoBench.GaokaoBench_no_subjective_gen_d21e37 import \
2115
GaokaoBench_datasets # noqa: F401, E501
22-
from opencompass.configs.datasets.gpqa.gpqa_few_shot_ppl_4b5a83 import \
23-
gpqa_datasets # noqa: F401, E501
24-
from opencompass.configs.datasets.gsm8k.gsm8k_gen_17d0dc import \
25-
gsm8k_datasets # noqa: F401, E501
16+
from opencompass.configs.datasets.gpqa.gpqa_few_shot_ppl_4b5a83 import gpqa_datasets # noqa: F401, E501
17+
from opencompass.configs.datasets.gsm8k.gsm8k_gen_17d0dc import gsm8k_datasets # noqa: F401, E501
2618
from opencompass.configs.datasets.hellaswag.hellaswag_10shot_ppl_59c85e import \
2719
hellaswag_datasets # noqa: F401, E501
2820
from opencompass.configs.datasets.humaneval.internal_humaneval_gen_ce6b06 import \
2921
humaneval_datasets as humaneval_v2_datasets # noqa: F401, E501
3022
from opencompass.configs.datasets.humaneval.internal_humaneval_gen_d2537e import \
3123
humaneval_datasets # noqa: F401, E501
32-
from opencompass.configs.datasets.math.math_4shot_base_gen_43d5b6 import \
33-
math_datasets # noqa: F401, E501
24+
from opencompass.configs.datasets.math.math_4shot_base_gen_43d5b6 import math_datasets # noqa: F401, E501
3425
from opencompass.configs.datasets.MathBench.mathbench_2024_few_shot_mixed_4a3fd4 import \
3526
mathbench_datasets # noqa: F401, E501
36-
from opencompass.configs.datasets.mbpp.sanitized_mbpp_gen_742f0c import \
37-
sanitized_mbpp_datasets # noqa: F401, E501
38-
from opencompass.configs.datasets.mmlu.mmlu_ppl_ac766d import \
39-
mmlu_datasets # noqa: F401, E501
40-
from opencompass.configs.datasets.mmlu_pro.mmlu_pro_few_shot_gen_bfaf90 import \
41-
mmlu_pro_datasets # noqa: F401, E501
42-
from opencompass.configs.datasets.nq.nq_open_1shot_gen_20a989 import \
43-
nq_datasets # noqa: F401, E501
44-
from opencompass.configs.datasets.race.race_few_shot_ppl import \
45-
race_datasets # noqa: F401, E501
27+
from opencompass.configs.datasets.mbpp.sanitized_mbpp_gen_742f0c import sanitized_mbpp_datasets # noqa: F401, E501
28+
from opencompass.configs.datasets.mmlu.mmlu_ppl_ac766d import mmlu_datasets # noqa: F401, E501
29+
from opencompass.configs.datasets.mmlu_pro.mmlu_pro_few_shot_gen_bfaf90 import mmlu_pro_datasets # noqa: F401, E501
30+
from opencompass.configs.datasets.nq.nq_open_1shot_gen_20a989 import nq_datasets # noqa: F401, E501
31+
from opencompass.configs.datasets.race.race_few_shot_ppl import race_datasets # noqa: F401, E501
4632
from opencompass.configs.datasets.SuperGLUE_BoolQ.SuperGLUE_BoolQ_few_shot_ppl import \
4733
BoolQ_datasets # noqa: F401, E501
48-
from opencompass.configs.datasets.TheoremQA.TheoremQA_5shot_gen_6f0af8 import \
49-
TheoremQA_datasets # noqa: F401, E501
34+
from opencompass.configs.datasets.TheoremQA.TheoremQA_5shot_gen_6f0af8 import TheoremQA_datasets # noqa: F401, E501
5035
from opencompass.configs.datasets.triviaqa.triviaqa_wiki_1shot_gen_20a989 import \
5136
triviaqa_datasets # noqa: F401, E501
5237
from opencompass.configs.datasets.wikibench.wikibench_few_shot_ppl_c23d79 import \
5338
wikibench_datasets # noqa: F401, E501
5439
from opencompass.configs.datasets.winogrande.winogrande_5shot_ll_252f01 import \
5540
winogrande_datasets # noqa: F401, E501
56-
from opencompass.configs.models.baichuan.hf_baichuan_7b import \
57-
models as hf_baichuan_7b # noqa: F401, E501
58-
from opencompass.configs.models.gemma.hf_gemma_7b import \
59-
models as hf_gemma_7b # noqa: F401, E501
60-
from opencompass.configs.models.hf_internlm.hf_internlm2_5_7b import \
61-
models as hf_internlm2_5_7b # noqa: F401, E501
62-
from opencompass.configs.models.hf_internlm.hf_internlm2_7b import \
63-
models as hf_internlm2_7b # noqa: F401, E501
64-
from opencompass.configs.models.hf_internlm.hf_internlm2_20b import \
65-
models as hf_internlm2_20b # noqa: F401, E501
66-
from opencompass.configs.models.hf_internlm.hf_internlm_7b import \
67-
models as hf_internlm_7b # noqa: F401, E501
68-
from opencompass.configs.models.hf_internlm.hf_internlm_20b import \
69-
models as hf_internlm_20b # noqa: F401, E501
41+
from opencompass.configs.models.baichuan.hf_baichuan_7b import models as hf_baichuan_7b # noqa: F401, E501
42+
from opencompass.configs.models.gemma.hf_gemma_7b import models as hf_gemma_7b # noqa: F401, E501
43+
from opencompass.configs.models.hf_internlm.hf_internlm2_5_7b import models as hf_internlm2_5_7b # noqa: F401, E501
44+
from opencompass.configs.models.hf_internlm.hf_internlm2_7b import models as hf_internlm2_7b # noqa: F401, E501
45+
from opencompass.configs.models.hf_internlm.hf_internlm2_20b import models as hf_internlm2_20b # noqa: F401, E501
46+
from opencompass.configs.models.hf_internlm.hf_internlm_7b import models as hf_internlm_7b # noqa: F401, E501
47+
from opencompass.configs.models.hf_internlm.hf_internlm_20b import models as hf_internlm_20b # noqa: F401, E501
7048
from opencompass.configs.models.hf_internlm.lmdeploy_internlm2_5_7b import \
7149
models as lmdeploy_internlm2_5_7b # noqa: F401, E501
72-
from opencompass.configs.models.hf_llama.hf_llama2_7b import \
73-
models as hf_llama2_7b # noqa: F401, E501
74-
from opencompass.configs.models.hf_llama.hf_llama3_8b import \
75-
models as hf_llama3_8b # noqa: F401, E501
76-
from opencompass.configs.models.mistral.hf_mistral_7b_v0_1 import \
77-
models as hf_mistral_7b_v0_1 # noqa: F401, E501
50+
from opencompass.configs.models.hf_llama.hf_llama2_7b import models as hf_llama2_7b # noqa: F401, E501
51+
from opencompass.configs.models.hf_llama.hf_llama3_8b import models as hf_llama3_8b # noqa: F401, E501
52+
from opencompass.configs.models.mistral.hf_mistral_7b_v0_1 import models as hf_mistral_7b_v0_1 # noqa: F401, E501
7853
from opencompass.configs.models.mistral.hf_mixtral_8x7b_v0_1 import \
7954
models as hf_mixtral_8x7b_v0_1 # noqa: F401, E501
80-
from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_7b import \
81-
models as lmdeploy_qwen2_5_7b # noqa: F401, E501
82-
from opencompass.configs.models.qwen.hf_qwen1_5_7b import \
83-
models as hf_qwen1_5_7b # noqa: F401, E501
84-
from opencompass.configs.models.qwen.hf_qwen2_7b import \
85-
models as hf_qwen2_7b # noqa: F401, E501
86-
from opencompass.configs.models.qwen.hf_qwen_7b import \
87-
models as hf_qwen_7b # noqa: F401, E501
88-
from opencompass.configs.models.qwen.lmdeploy_qwen1_5_7b import \
89-
models as lmdeploy_qwen1_5_7b # noqa: F401, E501
90-
from opencompass.configs.models.qwen.lmdeploy_qwen2_7b import \
91-
models as lmdeploy_qwen2_7b # noqa: F401, E501
55+
from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_7b import models as lmdeploy_qwen2_5_7b # noqa: F401, E501
56+
from opencompass.configs.models.qwen.hf_qwen1_5_7b import models as hf_qwen1_5_7b # noqa: F401, E501
57+
from opencompass.configs.models.qwen.hf_qwen2_7b import models as hf_qwen2_7b # noqa: F401, E501
58+
from opencompass.configs.models.qwen.hf_qwen_7b import models as hf_qwen_7b # noqa: F401, E501
59+
from opencompass.configs.models.qwen.lmdeploy_qwen1_5_7b import models as lmdeploy_qwen1_5_7b # noqa: F401, E501
60+
from opencompass.configs.models.qwen.lmdeploy_qwen2_7b import models as lmdeploy_qwen2_7b # noqa: F401, E501
9261
# Summary Groups
93-
from opencompass.configs.summarizers.groups.cmmlu import \
94-
cmmlu_summary_groups # noqa: F401, E501
95-
from opencompass.configs.summarizers.groups.GaokaoBench import \
96-
GaokaoBench_summary_groups # noqa: F401, E501
62+
from opencompass.configs.summarizers.groups.cmmlu import cmmlu_summary_groups # noqa: F401, E501
63+
from opencompass.configs.summarizers.groups.GaokaoBench import GaokaoBench_summary_groups # noqa: F401, E501
9764
from opencompass.configs.summarizers.groups.mathbench_v1_2024 import \
9865
mathbench_2024_summary_groups # noqa: F401, E501
99-
from opencompass.configs.summarizers.groups.mmlu import \
100-
mmlu_summary_groups # noqa: F401, E501
101-
from opencompass.configs.summarizers.groups.mmlu_pro import \
102-
mmlu_pro_summary_groups # noqa: F401, E501
66+
from opencompass.configs.summarizers.groups.mmlu import mmlu_summary_groups # noqa: F401, E501
67+
from opencompass.configs.summarizers.groups.mmlu_pro import mmlu_pro_summary_groups # noqa: F401, E501
10368

10469
# read models
10570
race_datasets = [race_datasets[1]]
@@ -169,8 +134,7 @@
169134
'mmlu_pro_psychology',
170135
'mmlu_pro_other',
171136
],
172-
summary_groups=sum(
173-
[v for k, v in locals().items() if k.endswith('_summary_groups')], []),
137+
summary_groups=sum([v for k, v in locals().items() if k.endswith('_summary_groups')], []),
174138
)
175139

176140
turbomind_qwen1_5_7b = deepcopy(*lmdeploy_qwen1_5_7b)

0 commit comments

Comments
 (0)