diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index cc694a9f70d..d01c3b8db06 100755 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -215,7 +215,11 @@ def main( exercises_dir: str = typer.Option( EXERCISES_DIR_DEFAULT, "--exercises-dir", help="Directory with exercise files" ), + timeout: Optional[int] = typer.Option( + None, "--timeout", help="Per-request API timeout in seconds (passed to litellm)." + ), ): + repo = git.Repo(search_parent_directories=True) commit_hash = repo.head.object.hexsha[:7] if repo.is_dirty(): @@ -350,6 +354,10 @@ def get_exercise_dirs(base_dir, languages=None): base_coder.RETRY_TIMEOUT = LONG_TIMEOUT models.RETRY_TIMEOUT = LONG_TIMEOUT + # Request timeout - default is 600, increase to avoid request timed out + if timeout: + models.request_timeout = timeout + if threads == 1: all_results = [] for test_path in test_dnames: