Skip to content

Commit dc85dad

Browse files
AnthonyDiGirolamoAnthonyDiGirolamo
authored
Use Pigweed's new Python build system (#21202)
This change updates Matter to use Pigweed's new Python build system. This reduces the number of pip installs performed during bootstrap and builds to just a single pigweed package along with any others specified in //BUILD.gn. All python_actions performed duing a GN build do not rely on any pip install of any in-tree python packages including Pigweed ones. Instead build a venv is created in the out directory with only 3rd party deps installed. All Python imports of in-tree packages are accomplished by setting PYTHONPATH for each python_action. This allows scaling up the number of in-tree Python packages without a significant impact to build speed. Bootstrap time is slightly reduced however all Pigweed Python modules are installed allong with the Python packages from: - //examples/common/pigweed/rpc_console - //examples/chef - //integrations/mobly Before: ``` Setting up Python environment.....done (1m34.7s) ``` After: ``` Setting up Python environment.....done (1m14.7s) ```
1 parent b6859d7 commit dc85dad

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+403
-211
lines changed

‎.github/workflows/full-android.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ jobs:
7272
- name: Run Android build rule tests
7373
run: |
7474
./scripts/run_in_build_env.sh \
75-
"ninja -C out/android-arm64-chip-tool build/chip/java/tests:java_build_test.tests"
75+
"ninja -C out/android-arm64-chip-tool build/chip/java/tests:java_build_test"
7676
- name: Clean out build output
7777
run: rm -rf ./out
7878
# - name: Build Android Studio build (arm64 only)

‎.github/workflows/smoketest-android.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -67,4 +67,4 @@ jobs:
6767
- name: Run Android build rule tests
6868
run: |
6969
./scripts/run_in_build_env.sh \
70-
"ninja -C out/android-arm64-chip-tool build/chip/java/tests:java_build_test.tests"
70+
"ninja -C out/android-arm64-chip-tool build/chip/java/tests:java_build_test"

‎.gn

+9-1
Original file line numberDiff line numberDiff line change
@@ -25,5 +25,13 @@ script_executable = "python3"
2525

2626
default_args = {
2727
pw_unit_test_AUTOMATIC_RUNNER = "$dir_pigweed/targets/host/run_test"
28-
pw_build_PIP_CONSTRAINTS = ["//scripts/constraints.txt"]
28+
29+
pw_build_PIP_CONSTRAINTS = [ "//scripts/constraints.txt" ]
30+
pw_build_PIP_REQUIREMENTS = [ "//scripts/requirements.txt" ]
31+
32+
# Use the new Python build and merged 'pigweed' Python package.
33+
pw_build_USE_NEW_PYTHON_BUILD = true
34+
35+
# GN target to use for the default Python build venv.
36+
pw_build_PYTHON_BUILD_VENV = "//:matter_build_venv"
2937
}

‎BUILD.gn

+76-18
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ import("//build_overrides/pigweed.gni")
2323
import("//src/lwip/lwip.gni")
2424
import("//src/platform/device.gni")
2525
import("$dir_pw_build/python.gni")
26+
import("$dir_pw_build/python_dist.gni")
27+
import("$dir_pw_build/python_venv.gni")
2628

2729
# This build file should not be used in superproject builds.
2830
assert(chip_root == "//")
@@ -53,29 +55,85 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") {
5355
}
5456
}
5557

56-
# Python packages for supporting specific targets.
58+
# Pigweed Python packages expected to be used in the :matter_build_venv
59+
# target. If all packages are needed this list should match
60+
# _pigweed_python_deps in:
61+
# https://cs.opensource.google/pigweed/pigweed/+/master:pw_env_setup/BUILD.gn?q=_pigweed_python_deps
62+
_pigweed_python_packages = [
63+
"$dir_pw_allocator/py",
64+
"$dir_pw_arduino_build/py",
65+
"$dir_pw_bloat/py",
66+
"$dir_pw_build/py",
67+
"$dir_pw_build_info/py",
68+
"$dir_pw_build_mcuxpresso/py",
69+
"$dir_pw_cli/py",
70+
"$dir_pw_console/py",
71+
"$dir_pw_cpu_exception_cortex_m/py",
72+
"$dir_pw_docgen/py",
73+
"$dir_pw_doctor/py",
74+
"$dir_pw_env_setup/py",
75+
"$dir_pw_hdlc/py",
76+
"$dir_pw_log:protos.python",
77+
"$dir_pw_log_tokenized/py",
78+
"$dir_pw_metric/py",
79+
"$dir_pw_module/py",
80+
"$dir_pw_package/py",
81+
"$dir_pw_presubmit/py",
82+
"$dir_pw_protobuf/py",
83+
"$dir_pw_protobuf_compiler/py",
84+
"$dir_pw_rpc/py",
85+
"$dir_pw_snapshot/py:pw_snapshot",
86+
"$dir_pw_snapshot/py:pw_snapshot_metadata",
87+
"$dir_pw_software_update/py",
88+
"$dir_pw_status/py",
89+
"$dir_pw_stm32cube_build/py",
90+
"$dir_pw_symbolizer/py",
91+
"$dir_pw_system/py",
92+
"$dir_pw_thread/py",
93+
"$dir_pw_tls_client/py",
94+
"$dir_pw_tokenizer/py",
95+
"$dir_pw_toolchain/py",
96+
"$dir_pw_trace/py",
97+
"$dir_pw_trace_tokenized/py",
98+
"$dir_pw_transfer/py",
99+
"$dir_pw_unit_test/py",
100+
"$dir_pw_watch/py",
101+
]
102+
103+
# Matter's in-tree pw_python_package or pw_create_python_source_tree targets.
104+
_matter_python_packages = [
105+
"//integrations/mobly:chip_mobly",
106+
"//examples/chef",
107+
"//examples/common/pigweed/rpc_console/py:chip_rpc",
108+
]
109+
110+
pw_python_venv("matter_build_venv") {
111+
path = "$root_build_dir/python-venv"
112+
constraints = pw_build_PIP_CONSTRAINTS
113+
requirements = pw_build_PIP_REQUIREMENTS
114+
115+
# Packages available to import within GN's build venv.
116+
source_packages = _matter_python_packages + _pigweed_python_packages
117+
}
118+
119+
# Python packages installed during bootstrap.
57120
pw_python_group("python_packages") {
58121
python_deps = [
59-
"$dir_pw_build/py",
60-
"$dir_pw_doctor/py",
61-
"$dir_pw_env_setup/py",
62-
"$dir_pw_hdlc/py",
63-
"$dir_pw_log:protos.python",
64-
"$dir_pw_module/py",
65-
"$dir_pw_protobuf/py",
66-
"$dir_pw_protobuf_compiler/py",
67-
"$dir_pw_rpc/py",
68-
"$dir_pw_status/py",
69-
"$dir_pw_toolchain/py",
70-
"$dir_pw_trace/py",
71-
"$dir_pw_trace_tokenized/py",
72-
"$dir_pw_unit_test/py",
73-
"$dir_pw_watch/py",
74-
"integrations/mobly:chip_mobly",
75-
"scripts:requirements",
122+
":pip_install_editable_matter_packages",
123+
"$dir_pw_env_setup:pip_install_pigweed_package",
76124
]
77125
}
78126

127+
# These pw_python_package targets will be installed using 'pip install --editable'
128+
pw_internal_pip_install("pip_install_editable_matter_packages") {
129+
packages = [
130+
"//integrations/mobly:chip_mobly",
131+
"//examples/chef",
132+
"//examples/common/pigweed/rpc_console/py:chip_rpc",
133+
]
134+
editable = true
135+
}
136+
79137
# This is a real toolchain. Build CHIP.
80138
group("default") {
81139
deps = [

‎build/chip/chip_test.gni

+5-1
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,11 @@ if (chip_link_tests) {
4444
pw_python_action(_test_name + "_run") {
4545
deps = [ ":${_test_name}" ]
4646
inputs = [ pw_unit_test_AUTOMATIC_RUNNER ]
47-
script = "$dir_pw_unit_test/py/pw_unit_test/test_runner.py"
47+
module = "pw_unit_test.test_runner"
48+
python_deps = [
49+
"$dir_pw_cli/py",
50+
"$dir_pw_unit_test/py",
51+
]
4852
args = [
4953
"--runner",
5054
rebase_path(pw_unit_test_AUTOMATIC_RUNNER, root_build_dir),

‎build/chip/java/tests/BUILD.gn

+4-3
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ import("//build_overrides/pigweed.gni")
1717
import("$dir_pw_build/python.gni")
1818
import("${chip_root}/build/chip/java/rules.gni")
1919

20-
pw_python_script("java_build_test") {
20+
pw_python_action("java_build_test") {
2121
inputs = [
2222
"expected_output/child_library_2_expected.json",
2323
"expected_output/grandchild_library_expected.json",
@@ -26,8 +26,9 @@ pw_python_script("java_build_test") {
2626
"expected_output/child_prebuilt_expected.json",
2727
"expected_output/java_prebuilt_expected.json",
2828
]
29-
other_deps = [ ":java_library" ]
30-
tests = [ "test.py" ]
29+
deps = [ ":java_library" ]
30+
script = "test.py"
31+
stamp = true
3132
}
3233

3334
java_library("java_library") {

‎build/chip/java/tests/expected_output/child_library_2_expected.json

+3-5
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,8 @@
11
{
22
"deps_info": {
33
"name": "child_library_2.json",
4-
"jar_path": "python/lib/build/chip/java/tests/child_library_2.jar",
5-
"deps_configs": [
6-
"python/gen/build/chip/java/tests/grandchild_library.json"
7-
],
8-
"deps_jars": ["python/lib/build/chip/java/tests/grandchild_library.jar"]
4+
"jar_path": "lib/build/chip/java/tests/child_library_2.jar",
5+
"deps_configs": ["gen/build/chip/java/tests/grandchild_library.json"],
6+
"deps_jars": ["lib/build/chip/java/tests/grandchild_library.jar"]
97
}
108
}

‎build/chip/java/tests/expected_output/child_library_expected.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"deps_info": {
33
"name": "child_library.json",
4-
"jar_path": "python/lib/build/chip/java/tests/child_library.jar",
4+
"jar_path": "lib/build/chip/java/tests/child_library.jar",
55
"deps_configs": [],
66
"deps_jars": []
77
}

‎build/chip/java/tests/expected_output/child_prebuilt_expected.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"deps_info": {
33
"name": "child_prebuilt.json",
4-
"jar_path": "python/lib/build/chip/java/tests/child_jar.jar",
4+
"jar_path": "lib/build/chip/java/tests/child_jar.jar",
55
"deps_configs": [],
66
"deps_jars": []
77
}

‎build/chip/java/tests/expected_output/grandchild_library_expected.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"deps_info": {
33
"name": "grandchild_library.json",
4-
"jar_path": "python/lib/build/chip/java/tests/grandchild_library.jar",
4+
"jar_path": "lib/build/chip/java/tests/grandchild_library.jar",
55
"deps_configs": [],
66
"deps_jars": []
77
}

‎build/chip/java/tests/expected_output/java_library_expected.json

+9-9
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,18 @@
11
{
22
"deps_info": {
33
"name": "java_library.json",
4-
"jar_path": "python/lib/build/chip/java/tests/java_library.jar",
4+
"jar_path": "lib/build/chip/java/tests/java_library.jar",
55
"deps_configs": [
6-
"python/gen/build/chip/java/tests/child_library.json",
7-
"python/gen/build/chip/java/tests/child_library_2.json",
8-
"python/gen/build/chip/java/tests/java_prebuilt.json"
6+
"gen/build/chip/java/tests/child_library.json",
7+
"gen/build/chip/java/tests/child_library_2.json",
8+
"gen/build/chip/java/tests/java_prebuilt.json"
99
],
1010
"deps_jars": [
11-
"python/lib/build/chip/java/tests/child_library.jar",
12-
"python/lib/build/chip/java/tests/child_library_2.jar",
13-
"python/lib/build/chip/java/tests/grandchild_library.jar",
14-
"python/lib/build/chip/java/tests/prebuilt_jar.jar",
15-
"python/lib/build/chip/java/tests/child_jar.jar"
11+
"lib/build/chip/java/tests/child_library.jar",
12+
"lib/build/chip/java/tests/child_library_2.jar",
13+
"lib/build/chip/java/tests/grandchild_library.jar",
14+
"lib/build/chip/java/tests/prebuilt_jar.jar",
15+
"lib/build/chip/java/tests/child_jar.jar"
1616
]
1717
}
1818
}

‎build/chip/java/tests/expected_output/java_prebuilt_expected.json

+3-5
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,8 @@
11
{
22
"deps_info": {
33
"name": "java_prebuilt.json",
4-
"jar_path": "python/lib/build/chip/java/tests/prebuilt_jar.jar",
5-
"deps_configs": [
6-
"python/gen/build/chip/java/tests/child_prebuilt.json"
7-
],
8-
"deps_jars": ["python/lib/build/chip/java/tests/child_jar.jar"]
4+
"jar_path": "lib/build/chip/java/tests/prebuilt_jar.jar",
5+
"deps_configs": ["gen/build/chip/java/tests/child_prebuilt.json"],
6+
"deps_jars": ["lib/build/chip/java/tests/child_jar.jar"]
97
}
108
}

‎build/chip/java/tests/test.py

+15-15
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1313
# See the License for the specific language governing permissions and
1414
# limitations under the License.
15-
1615
"""
1716
Test for GN Java build rules. This test should be executed using ninja.
1817
"""
@@ -29,8 +28,8 @@ class JavaBuildTest(unittest.TestCase):
2928
local_test_dir = '/build/chip/java/tests'
3029
test_dir = chip_root + local_test_dir
3130

32-
jars_dir = 'python/lib' + local_test_dir
33-
configs_dir = 'python/gen' + local_test_dir
31+
jars_dir = 'lib' + local_test_dir
32+
configs_dir = 'gen' + local_test_dir
3433

3534
# Target names in the BUILD.gn
3635
targets_to_check = [
@@ -39,10 +38,7 @@ class JavaBuildTest(unittest.TestCase):
3938
'child_library_2',
4039
'grandchild_library',
4140
]
42-
prebuilt_targets_to_check = [
43-
'java_prebuilt',
44-
'child_prebuilt'
45-
]
41+
prebuilt_targets_to_check = ['java_prebuilt', 'child_prebuilt']
4642

4743
def testExpectedJarsCreated(self):
4844
jars_dir = JavaBuildTest.jars_dir
@@ -58,18 +54,22 @@ def testBuildConfigMatchesExpected(self):
5854
configs_dir = JavaBuildTest.configs_dir
5955
expected_dir = JavaBuildTest.test_dir + '/expected_output'
6056

61-
for target in (JavaBuildTest.targets_to_check + JavaBuildTest.prebuilt_targets_to_check):
62-
with open(expected_dir + '/' + target + '_expected.json', 'r') as expected_config, open(configs_dir + '/' + target + '.json', 'r') as actual_config:
57+
for target in (JavaBuildTest.targets_to_check +
58+
JavaBuildTest.prebuilt_targets_to_check):
59+
with open(expected_dir + '/' + target + '_expected.json',
60+
'r') as expected_config, open(
61+
configs_dir + '/' + target + '.json',
62+
'r') as actual_config:
6363
expected_json = json.load(expected_config)['deps_info']
6464
actual_json = json.load(actual_config)['deps_info']
6565

6666
self.assertEqual(expected_json['name'], actual_json['name'])
67-
self.assertEqual(
68-
expected_json['jar_path'], actual_json['jar_path'])
69-
self.assertCountEqual(
70-
expected_json['deps_configs'], actual_json['deps_configs'])
71-
self.assertCountEqual(
72-
expected_json['deps_jars'], actual_json['deps_jars'])
67+
self.assertEqual(expected_json['jar_path'],
68+
actual_json['jar_path'])
69+
self.assertCountEqual(expected_json['deps_configs'],
70+
actual_json['deps_configs'])
71+
self.assertCountEqual(expected_json['deps_jars'],
72+
actual_json['deps_jars'])
7373

7474

7575
if __name__ == '__main__':

‎examples/build_overrides/pigweed_environment.gni

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,4 +26,4 @@ dir_cipd_pigweed =
2626
dir_cipd_python =
2727
get_path_info("${_bootstrap_root}/${dir_cipd_python}", "abspath")
2828
dir_virtual_env =
29-
get_path_info("${_bootstrap_root}/${dir_virtual_env}", "abspath")
29+
get_path_info("${_bootstrap_root}/${pw_env_setup_VIRTUAL_ENV}", "abspath")

‎examples/chef/BUILD.gn

+5-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,11 @@ import("//build_overrides/pigweed.gni")
1919
import("$dir_pw_build/python.gni")
2020

2121
pw_python_package("chef") {
22-
setup = [ "setup.py" ]
22+
setup = [
23+
"pyproject.toml",
24+
"setup.cfg",
25+
"setup.py",
26+
]
2327

2428
inputs = [
2529
"sample_app_util/test_files/sample_zap_file.zap",

‎examples/chef/README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@ chef_$PLATFORM:
148148
- name: Checkout submodules
149149
run: scripts/checkout_submodules.py --shallow --platform $PLATFORM
150150
- name: Bootstrap
151-
timeout-minutes: 10
151+
timeout-minutes: 25
152152
run: scripts/build/gn_bootstrap.sh
153153
- name: CI Examples $PLATFORM
154154
shell: bash

‎examples/chef/chef.py

+5-3
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,8 @@ def bundle_esp32(device_name: str) -> None:
239239
shutil.copy(src_item, dest_item)
240240

241241

242-
def main(argv: Sequence[str]) -> None:
242+
def main() -> int:
243+
243244
check_python_version()
244245
config = load_config()
245246
cicd_config = load_cicd_config()
@@ -340,7 +341,7 @@ def main(argv: Sequence[str]) -> None:
340341
parser.add_option(
341342
"", "--cpu_type", help="CPU type to compile for. Linux only.", choices=["arm64", "arm", "x64"])
342343

343-
options, _ = parser.parse_args(argv)
344+
options, _ = parser.parse_args(sys.argv[1:])
344345

345346
splash()
346347

@@ -795,7 +796,8 @@ def main(argv: Sequence[str]) -> None:
795796
shell.run_cmd(f"python3 -m chip_rpc.console --device {config['silabs-thread']['CU']} -b 115200")
796797

797798
flush_print("Done")
799+
return 0
798800

799801

800802
if __name__ == '__main__':
801-
sys.exit(main(sys.argv[1:]))
803+
sys.exit(main())

‎examples/chef/pyproject.toml

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Copyright (c) 2022 Project CHIP Authors
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
[build-system]
15+
requires = ['setuptools', 'wheel']
16+
build-backend = 'setuptools.build_meta'

0 commit comments

Comments
 (0)