Skip to content

Commit ddc48d9

Browse files
arkqarkq
authored
Fix CCTRL tests on CI - CCTRL cluster is on endpoint 1 (#36874)
* Fix CCTRL tests on CI - CCTRL cluster is on endpoint 1 * Fail CI tests if any test cases were skipped * Add compatibility flag to mobile-device-test.py * Comment out skipped tests * Change executable mode * Improve --test-case command line option * Disable skipped test cases on TC_SWTCH * Disable TC_LVL_2_3 on CI * Make TestBdxTransfer quiet * Verify testing support before running actual tests * Add missing definition
1 parent 204fb27 commit ddc48d9

File tree

14 files changed

+85
-70
lines changed

14 files changed

+85
-70
lines changed

‎.github/workflows/tests.yaml

+17-15
Original file line numberDiff line numberDiff line change
@@ -520,24 +520,26 @@ jobs:
520520
echo "TRACE_TEST_JSON: out/trace_data/test-{SCRIPT_BASE_NAME}" >> /tmp/test_env.yaml
521521
echo "TRACE_TEST_PERFETTO: out/trace_data/test-{SCRIPT_BASE_NAME}" >> /tmp/test_env.yaml
522522
523+
- name: Verify Testing Support
524+
run: |
525+
scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/test_testing/test_IDM_10_4.py'
526+
scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/test_testing/test_TC_ICDM_2_1.py'
527+
scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/test_testing/test_TC_SC_7_1.py'
528+
scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/test_testing/TestDecorators.py'
529+
scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/TestChoiceConformanceSupport.py'
530+
scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/TestConformanceSupport.py'
531+
scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/TestConformanceTest.py'
532+
scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/TestIdChecks.py'
533+
scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/TestMatterTestingSupport.py'
534+
scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/TestSpecParsingDeviceType.py'
535+
scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/TestSpecParsingSupport.py'
536+
523537
- name: Run Tests
524538
run: |
525539
mkdir -p out/trace_data
526-
scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --load-from-env /tmp/test_env.yaml --script src/controller/python/test/test_scripts/mobile-device-test.py'
527-
scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/execute_python_tests.py --env-file /tmp/test_env.yaml --search-directory src/python_testing'
528-
scripts/run_in_python_env.sh out/venv './scripts/tests/TestTimeSyncTrustedTimeSourceRunner.py --all-clusters out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app'
529-
scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/TestIdChecks.py'
530-
scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/TestSpecParsingDeviceType.py'
531-
scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/TestConformanceSupport.py'
532-
scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/TestConformanceTest.py'
533-
scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/TestChoiceConformanceSupport.py'
534-
scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/TestMatterTestingSupport.py'
535-
scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/TestSpecParsingSupport.py'
536-
scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/test_testing/test_TC_ICDM_2_1.py'
537-
scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/test_testing/test_IDM_10_4.py'
538-
scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/test_testing/test_TC_SC_7_1.py'
539-
scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/test_testing/TestDecorators.py'
540-
540+
scripts/run_in_python_env.sh out/venv 'scripts/tests/run_python_test.py --load-from-env /tmp/test_env.yaml --script src/controller/python/test/test_scripts/mobile-device-test.py'
541+
scripts/run_in_python_env.sh out/venv 'src/python_testing/execute_python_tests.py --env-file /tmp/test_env.yaml --search-directory src/python_testing'
542+
scripts/run_in_python_env.sh out/venv 'scripts/tests/TestTimeSyncTrustedTimeSourceRunner.py --all-clusters out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app'
541543
542544
- name: Uploading core files
543545
uses: actions/upload-artifact@v4

‎scripts/tests/run_python_test.py

+7-2
Original file line numberDiff line numberDiff line change
@@ -215,8 +215,13 @@ def main_impl(app: str, factory_reset: bool, factory_reset_app_only: bool, app_a
215215
app_process.p.stdin.close()
216216
app_pid = app_process.p.pid
217217

218-
script_command = [script, "--paa-trust-store-path", os.path.join(DEFAULT_CHIP_ROOT, MATTER_DEVELOPMENT_PAA_ROOT_CERTS),
219-
'--log-format', '%(message)s', "--app-pid", str(app_pid)] + shlex.split(script_args)
218+
script_command = [
219+
script,
220+
"--fail-on-skipped",
221+
"--paa-trust-store-path", os.path.join(DEFAULT_CHIP_ROOT, MATTER_DEVELOPMENT_PAA_ROOT_CERTS),
222+
"--log-format", '%(message)s',
223+
"--app-pid", str(app_pid),
224+
] + shlex.split(script_args)
220225

221226
if script_gdb:
222227
#

‎src/controller/python/test/test_scripts/mobile-device-test.py

+4-1
Original file line numberDiff line numberDiff line change
@@ -266,8 +266,11 @@ def do_tests(controller_nodeid, device_nodeid, address, timeout, discriminator,
266266
type=int,
267267
default=0,
268268
help="The PID of the app against which the test is going to run")
269+
@click.option('--fail-on-skipped',
270+
is_flag=True,
271+
help="Fail the test if any test cases are skipped")
269272
def run(controller_nodeid, device_nodeid, address, timeout, discriminator, setup_pin, enable_test, disable_test, log_level,
270-
log_format, print_test_list, paa_trust_store_path, trace_to, app_pid):
273+
log_format, print_test_list, paa_trust_store_path, trace_to, app_pid, fail_on_skipped):
271274
coloredlogs.install(level=log_level, fmt=log_format, logger=logger)
272275

273276
if print_test_list:

‎src/python_testing/TC_CCTRL_2_1.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
# --commissioning-method on-network
3232
# --discriminator 1234
3333
# --passcode 20202021
34-
# --endpoint 0
34+
# --endpoint 1
3535
# --trace-to json:${TRACE_TEST_JSON}.json
3636
# --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
3737
# factory-reset: true
@@ -46,7 +46,7 @@
4646
# --commissioning-method on-network
4747
# --discriminator 1234
4848
# --passcode 20202021
49-
# --endpoint 0
49+
# --endpoint 1
5050
# --trace-to json:${TRACE_TEST_JSON}.json
5151
# --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
5252
# factory-reset: true

‎src/python_testing/TC_CCTRL_2_2.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
# --commissioning-method on-network
3232
# --discriminator 1234
3333
# --passcode 20202021
34-
# --endpoint 0
34+
# --endpoint 1
3535
# --string-arg th_server_app_path:${ALL_CLUSTERS_APP}
3636
# --trace-to json:${TRACE_TEST_JSON}.json
3737
# --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
@@ -47,7 +47,7 @@
4747
# --commissioning-method on-network
4848
# --discriminator 1234
4949
# --passcode 20202021
50-
# --endpoint 0
50+
# --endpoint 1
5151
# --string-arg th_server_app_path:${ALL_CLUSTERS_APP}
5252
# --trace-to json:${TRACE_TEST_JSON}.json
5353
# --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto

‎src/python_testing/TC_CCTRL_2_3.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
# --commissioning-method on-network
3232
# --discriminator 1234
3333
# --passcode 20202021
34-
# --endpoint 0
34+
# --endpoint 1
3535
# --string-arg th_server_app_path:${ALL_CLUSTERS_APP}
3636
# --trace-to json:${TRACE_TEST_JSON}.json
3737
# --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
@@ -47,7 +47,7 @@
4747
# --commissioning-method on-network
4848
# --discriminator 1234
4949
# --passcode 20202021
50-
# --endpoint 0
50+
# --endpoint 1
5151
# --string-arg th_server_app_path:${ALL_CLUSTERS_APP}
5252
# --trace-to json:${TRACE_TEST_JSON}.json
5353
# --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto

‎src/python_testing/TC_DeviceBasicComposition.py

+14-15
Original file line numberDiff line numberDiff line change
@@ -189,7 +189,6 @@
189189
find_tag_list_problems, find_tree_roots, flat_list_ok,
190190
get_direct_children_of_root, parts_list_cycles, separate_endpoint_types)
191191
from chip.tlv import uint
192-
from mobly import asserts
193192

194193

195194
def get_vendor_id(mei: int) -> int:
@@ -692,25 +691,25 @@ def test_TC_IDM_11_1(self):
692691
if not success:
693692
self.fail_current_test("At least one attribute string was not valid UTF-8")
694693

695-
def test_all_event_strings_valid(self):
696-
asserts.skip("TODO: Validate every string in the read events is valid UTF-8 and has no nulls")
694+
# def test_all_event_strings_valid(self):
695+
# asserts.skip("TODO: Validate every string in the read events is valid UTF-8 and has no nulls")
697696

698-
def test_all_schema_scalars(self):
699-
asserts.skip("TODO: Validate all int/uint are in range of the schema (or null if nullable) for known attributes")
697+
# def test_all_schema_scalars(self):
698+
# asserts.skip("TODO: Validate all int/uint are in range of the schema (or null if nullable) for known attributes")
700699

701-
def test_all_commands_reported_are_executable(self):
702-
asserts.skip("TODO: Validate all commands reported in AcceptedCommandList are actually executable")
700+
# def test_all_commands_reported_are_executable(self):
701+
# asserts.skip("TODO: Validate all commands reported in AcceptedCommandList are actually executable")
703702

704-
def test_dump_all_pics_for_all_endpoints(self):
705-
asserts.skip("TODO: Make a test that generates the basic PICS list for each endpoint based on actually reported contents")
703+
# def test_dump_all_pics_for_all_endpoints(self):
704+
# asserts.skip("TODO: Make a test that generates the basic PICS list for each endpoint based on actually reported contents")
706705

707-
def test_all_schema_mandatory_elements_present(self):
708-
asserts.skip(
709-
"TODO: Make a test that ensures every known cluster has the mandatory elements present (commands, attributes) based on features")
706+
# def test_all_schema_mandatory_elements_present(self):
707+
# asserts.skip(
708+
# "TODO: Make a test that ensures every known cluster has the mandatory elements present (commands, attributes) based on features")
710709

711-
def test_all_endpoints_have_valid_composition(self):
712-
asserts.skip(
713-
"TODO: Make a test that verifies each endpoint has valid set of device types, and that the device type conformance is respected for each")
710+
# def test_all_endpoints_have_valid_composition(self):
711+
# asserts.skip(
712+
# "TODO: Make a test that verifies each endpoint has valid set of device types, and that the device type conformance is respected for each")
714713

715714
def test_TC_SM_1_2(self):
716715
self.print_step(1, "Wildcard read of device - already done")

‎src/python_testing/TC_LVL_2_3.py

+1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
1919
# for details about the block below.
2020
#
21+
# FIXME: https://github.com/project-chip/connectedhomeip/issues/36885
2122
# === BEGIN CI TEST ARGUMENTS ===
2223
# test-runner-runs:
2324
# run1:

‎src/python_testing/TC_SWTCH.py

+12-16
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,15 @@
1717
# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
1818
# for details about the block below.
1919
#
20+
# TODO: https://github.com/project-chip/connectedhomeip/issues/36884
21+
#
2022
# === BEGIN CI TEST ARGUMENTS ===
2123
# test-runner-runs:
2224
# run1:
2325
# app: ${ALL_CLUSTERS_APP}
2426
# app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
2527
# script-args: >
28+
# --test-case test_TC_SWTCH_2_2
2629
# --endpoint 1
2730
# --storage-path admin_storage.json
2831
# --commissioning-method on-network
@@ -37,20 +40,9 @@
3740
# app: ${ALL_CLUSTERS_APP}
3841
# app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
3942
# script-args: >
40-
# --endpoint 2
41-
# --storage-path admin_storage.json
42-
# --commissioning-method on-network
43-
# --discriminator 1234
44-
# --passcode 20202021
45-
# --trace-to json:${TRACE_TEST_JSON}.json
46-
# --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
47-
# --PICS src/app/tests/suites/certification/ci-pics-values
48-
# factory-reset: true
49-
# quiet: true
50-
# run3:
51-
# app: ${ALL_CLUSTERS_APP}
52-
# app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
53-
# script-args: >
43+
# --test-case test_TC_SWTCH_2_3
44+
# --test-case test_TC_SWTCH_2_4
45+
# --test-case test_TC_SWTCH_2_6
5446
# --endpoint 3
5547
# --storage-path admin_storage.json
5648
# --commissioning-method on-network
@@ -61,10 +53,13 @@
6153
# --PICS src/app/tests/suites/certification/ci-pics-values
6254
# factory-reset: true
6355
# quiet: true
64-
# run4:
56+
# run3:
6557
# app: ${ALL_CLUSTERS_APP}
6658
# app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
6759
# script-args: >
60+
# --test-case test_TC_SWTCH_2_3
61+
# --test-case test_TC_SWTCH_2_4
62+
# --test-case test_TC_SWTCH_2_5
6863
# --endpoint 4
6964
# --storage-path admin_storage.json
7065
# --commissioning-method on-network
@@ -281,7 +276,8 @@ def _expect_no_events_for_cluster(self, event_queue: queue.Queue, endpoint_id: i
281276
elapsed = 0.0
282277
time_remaining = timeout_sec
283278

284-
logging.info(f"Waiting {timeout_sec:.1f} seconds for no more events for cluster {expected_cluster} on endpoint {endpoint_id}")
279+
logging.info(f"Waiting {timeout_sec:.1f} seconds for no more events for "
280+
f"cluster {expected_cluster} on endpoint {endpoint_id}")
285281
while time_remaining > 0:
286282
try:
287283
item: EventReadResult = event_queue.get(block=True, timeout=time_remaining)

‎src/python_testing/TestBdxTransfer.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
# --trace-to json:${TRACE_TEST_JSON}.json
3333
# --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
3434
# factory-reset: true
35-
# quiet: false
35+
# quiet: true
3636
# === END CI TEST ARGUMENTS ===
3737

3838
import asyncio

‎src/python_testing/TestMatterTestingSupport.py

+3
Original file line numberDiff line numberDiff line change
@@ -640,6 +640,8 @@ def test_xml_pics(self):
640640

641641
def test_parse_matter_test_args(self):
642642
args = [
643+
# Verify that it is possible to pass multiple test cases at once
644+
"--tests", "TC_1", "TC_2",
643645
# Verify that values are appended to a single argument
644646
"--int-arg", "PIXIT.TEST.DEC:42",
645647
"--int-arg", "PIXIT.TEST.HEX:0x1234",
@@ -650,6 +652,7 @@ def test_parse_matter_test_args(self):
650652
]
651653

652654
parsed = parse_matter_test_args(args)
655+
asserts.assert_equal(parsed.tests, ["TC_1", "TC_2"])
653656
asserts.assert_equal(parsed.global_test_params.get("PIXIT.TEST.DEC"), 42)
654657
asserts.assert_equal(parsed.global_test_params.get("PIXIT.TEST.HEX"), 0x1234)
655658
asserts.assert_equal(parsed.global_test_params.get("PIXIT.TEST.STR.MULTI.1"), "foo")

‎src/python_testing/execute_python_tests.py

100644100755
+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#!/usr/bin/env -S python3 -B
12
#
23
# Copyright (c) 2024 Project CHIP Authors
34
# All rights reserved.

‎src/python_testing/matter_testing_infrastructure/chip/testing/matter_testing.py

+14-13
Original file line numberDiff line numberDiff line change
@@ -628,6 +628,7 @@ class MatterTestConfig:
628628
timeout: typing.Union[int, None] = None
629629
endpoint: typing.Union[int, None] = 0
630630
app_pid: int = 0
631+
fail_on_skipped_tests: bool = False
631632

632633
commissioning_method: Optional[str] = None
633634
discriminators: List[int] = field(default_factory=list)
@@ -1932,10 +1933,11 @@ def convert_args_to_matter_config(args: argparse.Namespace) -> MatterTestConfig:
19321933
config.paa_trust_store_path = args.paa_trust_store_path
19331934
config.ble_interface_id = args.ble_interface_id
19341935
config.pics = {} if args.PICS is None else read_pics_from_file(args.PICS)
1935-
config.tests = [] if args.tests is None else args.tests
1936+
config.tests = list(chain.from_iterable(args.tests or []))
19361937
config.timeout = args.timeout # This can be none, we pull the default from the test if it's unspecified
19371938
config.endpoint = args.endpoint # This can be None, the get_endpoint function allows the tests to supply a default
19381939
config.app_pid = 0 if args.app_pid is None else args.app_pid
1940+
config.fail_on_skipped_tests = args.fail_on_skipped
19391941

19401942
config.controller_node_id = args.controller_node_id
19411943
config.trace_to = args.trace_to
@@ -1962,13 +1964,10 @@ def parse_matter_test_args(argv: Optional[List[str]] = None) -> MatterTestConfig
19621964

19631965
basic_group = parser.add_argument_group(title="Basic arguments", description="Overall test execution arguments")
19641966

1965-
basic_group.add_argument('--tests',
1966-
'--test_case',
1967-
action="store",
1968-
nargs='+',
1969-
type=str,
1970-
metavar='test_a test_b...',
1967+
basic_group.add_argument('--tests', '--test-case', action='append', nargs='+', type=str, metavar='test_NAME',
19711968
help='A list of tests in the test class to execute.')
1969+
basic_group.add_argument('--fail-on-skipped', action="store_true", default=False,
1970+
help="Fail the test if any test cases are skipped")
19721971
basic_group.add_argument('--trace-to', nargs="*", default=[],
19731972
help="Where to trace (e.g perfetto, perfetto:path, json:log, json:path)")
19741973
basic_group.add_argument('--storage-path', action="store", type=pathlib.Path,
@@ -2056,17 +2055,17 @@ def parse_matter_test_args(argv: Optional[List[str]] = None) -> MatterTestConfig
20562055
help='Path to chip-tool credentials file root')
20572056

20582057
args_group = parser.add_argument_group(title="Config arguments", description="Test configuration global arguments set")
2059-
args_group.add_argument('--int-arg', nargs='*', action='append', type=int_named_arg, metavar="NAME:VALUE",
2058+
args_group.add_argument('--int-arg', nargs='+', action='append', type=int_named_arg, metavar="NAME:VALUE",
20602059
help="Add a named test argument for an integer as hex or decimal (e.g. -2 or 0xFFFF_1234)")
2061-
args_group.add_argument('--bool-arg', nargs='*', action='append', type=bool_named_arg, metavar="NAME:VALUE",
2060+
args_group.add_argument('--bool-arg', nargs='+', action='append', type=bool_named_arg, metavar="NAME:VALUE",
20622061
help="Add a named test argument for an boolean value (e.g. true/false or 0/1)")
2063-
args_group.add_argument('--float-arg', nargs='*', action='append', type=float_named_arg, metavar="NAME:VALUE",
2062+
args_group.add_argument('--float-arg', nargs='+', action='append', type=float_named_arg, metavar="NAME:VALUE",
20642063
help="Add a named test argument for a floating point value (e.g. -2.1 or 6.022e23)")
2065-
args_group.add_argument('--string-arg', nargs='*', action='append', type=str_named_arg, metavar="NAME:VALUE",
2064+
args_group.add_argument('--string-arg', nargs='+', action='append', type=str_named_arg, metavar="NAME:VALUE",
20662065
help="Add a named test argument for a string value")
2067-
args_group.add_argument('--json-arg', nargs='*', action='append', type=json_named_arg, metavar="NAME:VALUE",
2066+
args_group.add_argument('--json-arg', nargs='+', action='append', type=json_named_arg, metavar="NAME:VALUE",
20682067
help="Add a named test argument for JSON stored as a list or dict")
2069-
args_group.add_argument('--hex-arg', nargs='*', action='append', type=bytes_as_hex_named_arg, metavar="NAME:VALUE",
2068+
args_group.add_argument('--hex-arg', nargs='+', action='append', type=bytes_as_hex_named_arg, metavar="NAME:VALUE",
20702069
help="Add a named test argument for an octet string in hex (e.g. 0011cafe or 00:11:CA:FE)")
20712070

20722071
if not argv:
@@ -2510,6 +2509,8 @@ def run_tests_no_exit(test_class: MatterBaseTest, matter_test_config: MatterTest
25102509
try:
25112510
runner.run()
25122511
ok = runner.results.is_all_pass and ok
2512+
if matter_test_config.fail_on_skipped_tests and runner.results.skipped:
2513+
ok = False
25132514
except TimeoutError:
25142515
ok = False
25152516
except signals.TestAbortAll:

0 commit comments

Comments
 (0)