From 9197701aa78b868370d00fe5822dab404e917daa Mon Sep 17 00:00:00 2001 From: James Graham Date: Wed, 7 Jan 2026 15:25:16 +0000 Subject: [PATCH 1/3] Convert run_tc.py into 'wpt tc-run' This makes it easier to run in a local docker image. --- tools/ci/commands.json | 8 ++++++ tools/ci/run_tc.py | 63 +++++++++++++++++++++-------------------- tools/ci/tc/decision.py | 2 +- 3 files changed, 42 insertions(+), 31 deletions(-) diff --git a/tools/ci/commands.json b/tools/ci/commands.json index 590d243abfb89b..79c51560c54dc8 100644 --- a/tools/ci/commands.json +++ b/tools/ci/commands.json @@ -80,6 +80,14 @@ "requirements_tc.txt" ] }, + "tc-run": { + "path": "run_tc.py", + "script": "run_tc", + "parser": "get_parser", + "parse_known": true, + "help": "Run command in Taskcluster CI environment (for use in Docker image)", + "virtualenv": false + }, "update-codeowners": { "path": "update_codeowners.py", "parser": "get_parser", diff --git a/tools/ci/run_tc.py b/tools/ci/run_tc.py index a729bc8878cf4f..02eb99fa529b36 100755 --- a/tools/ci/run_tc.py +++ b/tools/ci/run_tc.py @@ -256,33 +256,33 @@ def unpack(path): raise Exception -def setup_environment(args): +def setup_environment(**kwargs): if "TASK_ARTIFACTS" in os.environ: artifacts = json.loads(os.environ["TASK_ARTIFACTS"]) download_artifacts(artifacts) - if args.hosts_file: + if kwargs["hosts_file"]: make_hosts_file() - if args.install_certificates: + if kwargs["install_certificates"]: install_certificates() - if "chrome" in args.browser: - assert args.channel is not None - install_chrome(args.channel) + if "chrome" in kwargs["browser"]: + assert kwargs["channel"] is not None + install_chrome(kwargs["channel"]) # These browsers use dbus for various features. - if any(b in args.browser for b in ["chrome", "webkitgtk_minibrowser", "wpewebkit_minibrowser"]): + if any(b in kwargs["browser"] for b in ["chrome", "webkitgtk_minibrowser", "wpewebkit_minibrowser"]): start_dbus() - if args.xvfb: + if kwargs["xvfb"]: start_xvfb() - if args.oom_killer: + if kwargs["oom_killer"]: start_userspace_oom_killer() -def setup_repository(args): +def setup_repository(**kwargs): is_pr = os.environ.get("GITHUB_PULL_REQUEST", "false") != "false" # Initially task_head points at the same commit as the ref we want to test. @@ -303,12 +303,12 @@ def setup_repository(args): # resources. In the latter case we assume it's OK to use the current merge # instead of the one at the time the decision task ran. - if args.ref: + if kwargs["ref"]: if is_pr: - assert args.ref.endswith("/merge") - expected_head = args.merge_rev + assert kwargs["ref"].endswith("/merge") + expected_head = kwargs["merge_rev"] else: - expected_head = args.head_rev + expected_head = kwargs["head_rev"] task_head = run(["git", "rev-parse", "task_head"], return_stdout=True).strip() @@ -324,14 +324,14 @@ def setup_repository(args): sys.exit(1) else: # Convert the refs/pulls//merge to refs/pulls//head - head_ref = args.ref.rsplit("/", 1)[0] + "/head" + head_ref = kwargs["ref"].rsplit("/", 1)[0] + "/head" try: remote_head = run(["git", "ls-remote", "origin", head_ref], return_stdout=True).split("\t")[0] except subprocess.CalledProcessError: print("CRITICAL: Failed to read remote ref %s" % head_ref) sys.exit(1) - if remote_head != args.head_rev: + if remote_head != kwargs["head_rev"]: print("CRITICAL: task_head points at %s, expected %s. " "This may be because the branch was updated" % (task_head, expected_head)) sys.exit(1) @@ -362,7 +362,7 @@ def setup_repository(args): # TODO: move this somewhere earlier in the task run(["git", "fetch", "--quiet", "origin", "%s:%s" % (branch, branch)]) - checkout_rev = args.checkout if args.checkout is not None else "task_head" + checkout_rev = kwargs["checkout"] if kwargs["checkout"] is not None else "task_head" checkout_revision(checkout_rev) refs = run(["git", "for-each-ref", "refs/heads"], return_stdout=True) @@ -405,9 +405,7 @@ def include_job(job): return job in set(jobs_str.splitlines()) -def main(): - args = get_parser().parse_args() - +def run_tc(*args, **kwargs): if "TASK_EVENT" in os.environ: event = json.loads(os.environ["TASK_EVENT"]) else: @@ -416,26 +414,31 @@ def main(): if event: set_variables(event) - if args.setup_repository: - setup_repository(args) + if kwargs["setup_repository"]: + setup_repository(**kwargs) # Hack for backwards compatibility - if args.script in ["run-all", "lint", "update_built", "tools_unittest", - "wpt_integration", "resources_unittest", - "wptrunner_infrastructure", "stability", "affected_tests"]: - job = args.script + if kwargs["script"] in ["run-all", "lint", "update_built", "tools_unittest", + "wpt_integration", "resources_unittest", + "wptrunner_infrastructure", "stability", "affected_tests"]: + job = kwargs["script"] if not include_job(job): return - args.script = args.script_args[0] - args.script_args = args.script_args[1:] + kwargs["script"] = kwargs["script_args"][0] + kwargs["script_args"] = kwargs["script_args"][1:] # Run the job - setup_environment(args) + setup_environment(**kwargs) os.chdir(root) - cmd = [args.script] + args.script_args + cmd = [kwargs["script"]] + kwargs["script_args"] print(" ".join(cmd)) sys.exit(subprocess.call(cmd)) +def main(): + args = get_parser().parse_args() + run_tc(**vars(args)) + + if __name__ == "__main__": main() # type: ignore diff --git a/tools/ci/tc/decision.py b/tools/ci/tc/decision.py index d00ba6ba194ee1..d98f54811295a9 100644 --- a/tools/ci/tc/decision.py +++ b/tools/ci/tc/decision.py @@ -227,7 +227,7 @@ def build_full_command(event, task): %(fetch_ref)s; %(install_str)s cd web-platform-tests; -./tools/ci/run_tc.py %(options_str)s -- %(task_cmd)s; +./wpt tc-run %(options_str)s -- %(task_cmd)s; """ % cmd_args] From ec8972f7047c949e8290e5a83cf8b04d21adac5c Mon Sep 17 00:00:00 2001 From: James Graham Date: Wed, 7 Jan 2026 15:28:22 +0000 Subject: [PATCH 2/3] Detect if we're actually running under taskcluster in tc-run This makes it easier to just copy/paste a command from CI into a local docker image. --- tools/ci/run_tc.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tools/ci/run_tc.py b/tools/ci/run_tc.py index 02eb99fa529b36..dcf1006598ece0 100755 --- a/tools/ci/run_tc.py +++ b/tools/ci/run_tc.py @@ -99,6 +99,8 @@ def get_parser(): help="Install web-platform.test certificates to UA store") p.add_argument("--no-install-certificates", action="store_false", default=None, help="Don't install web-platform.test certificates to UA store") + p.add_argument("--setup-repository", action="store_true", default=None, dest="setup_repository", + help="Run any repository setup steps, instead use the existing worktree") p.add_argument("--no-setup-repository", action="store_false", dest="setup_repository", help="Don't run any repository setup steps, instead use the existing worktree. " "This is useful for local testing.") @@ -406,15 +408,21 @@ def include_job(job): def run_tc(*args, **kwargs): + is_ci = "TASKCLUSTER_ROOT_URL" in os.environ + if "TASK_EVENT" in os.environ: event = json.loads(os.environ["TASK_EVENT"]) - else: + if "TASK_EVENT" in os.environ: + event = json.loads(os.environ["TASK_EVENT"]) + elif is_ci: event = fetch_event_data() + else: + event = None if event: set_variables(event) - if kwargs["setup_repository"]: + if kwargs["setup_repository"] or (kwargs["setup_repository"] is None and is_ci): setup_repository(**kwargs) # Hack for backwards compatibility From 9d9cad52e713ea86faa26d87af16934646ea2f9b Mon Sep 17 00:00:00 2001 From: James Graham Date: Wed, 7 Jan 2026 15:29:20 +0000 Subject: [PATCH 3/3] Kill started processes when exiting from tc-run This isn't necessary in actual CI, but locally it makes it easier to run multiple commands in the same docker image. --- tools/ci/run_tc.py | 69 +++++++++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/tools/ci/run_tc.py b/tools/ci/run_tc.py index dcf1006598ece0..aa95618615d589 100755 --- a/tools/ci/run_tc.py +++ b/tools/ci/run_tc.py @@ -55,6 +55,9 @@ os.pardir)) +started_processes = [] + + def run(cmd, return_stdout=False, **kwargs): print(" ".join(cmd)) if return_stdout: @@ -68,7 +71,7 @@ def run(cmd, return_stdout=False, **kwargs): def start(cmd): print(" ".join(cmd)) - subprocess.Popen(cmd) + started_processes.append(subprocess.Popen(cmd)) def get_parser(): @@ -408,39 +411,41 @@ def include_job(job): def run_tc(*args, **kwargs): + try: is_ci = "TASKCLUSTER_ROOT_URL" in os.environ - if "TASK_EVENT" in os.environ: - event = json.loads(os.environ["TASK_EVENT"]) - if "TASK_EVENT" in os.environ: - event = json.loads(os.environ["TASK_EVENT"]) - elif is_ci: - event = fetch_event_data() - else: - event = None - - if event: - set_variables(event) - - if kwargs["setup_repository"] or (kwargs["setup_repository"] is None and is_ci): - setup_repository(**kwargs) - - # Hack for backwards compatibility - if kwargs["script"] in ["run-all", "lint", "update_built", "tools_unittest", - "wpt_integration", "resources_unittest", - "wptrunner_infrastructure", "stability", "affected_tests"]: - job = kwargs["script"] - if not include_job(job): - return - kwargs["script"] = kwargs["script_args"][0] - kwargs["script_args"] = kwargs["script_args"][1:] - - # Run the job - setup_environment(**kwargs) - os.chdir(root) - cmd = [kwargs["script"]] + kwargs["script_args"] - print(" ".join(cmd)) - sys.exit(subprocess.call(cmd)) + if "TASK_EVENT" in os.environ: + event = json.loads(os.environ["TASK_EVENT"]) + elif is_ci: + event = fetch_event_data() + else: + event = None + + if event: + set_variables(event) + + if kwargs["setup_repository"] or (kwargs["setup_repository"] is None and is_ci): + setup_repository(**kwargs) + + # Hack for backwards compatibility + if kwargs["script"] in ["run-all", "lint", "update_built", "tools_unittest", + "wpt_integration", "resources_unittest", + "wptrunner_infrastructure", "stability", "affected_tests"]: + job = kwargs["script"] + if not include_job(job): + return + kwargs["script"] = kwargs["script_args"][0] + kwargs["script_args"] = kwargs["script_args"][1:] + + # Run the job + setup_environment(**kwargs) + os.chdir(root) + cmd = [kwargs["script"]] + kwargs["script_args"] + print(" ".join(cmd)) + sys.exit(subprocess.call(cmd)) + finally: + for process in started_processes: + process.kill() def main():