Perform pip install as part of bootstrap.py

Also, install pip packages in local `out/python_deps` directory instead
of installing system wide.

This means we can consistently expect dev dependencies to be available
in test code without needing to include an opt out mechanism.

We were already doing this for `psutil`, but for `websockify` we made
it optional.

This means that only python scripts that explicitly add
`out/python_deps` to their python path will be able use
the packages, and in particular it means that the emscripten compiler
itself won't end up implicitly/accidentally depending on them.

Author: sbc100

.circleci/config.yml

@@ -61,15 +61,10 @@ commands:
       - run: ./bootstrap
   pip-install:
     description: "pip install"
-    parameters:
-      python:
-        description: "Python executable to use"
-        type: string
-        default: python3
     steps:
       - run:
           name: pip install
-          command: << parameters.python >> -m pip install -r requirements-dev.txt
+          command: python3 -m pip install -r requirements-dev.txt
   install-rust:
     steps:
       - run:
@@ -186,7 +181,6 @@ commands:
           name: clear cache
           command: |
             ./emcc --clear-cache
-      - pip-install
       - run: apt-get install -q -y ninja-build ccache
       - run:
           name: Ccache stats and configuration
@@ -266,7 +260,6 @@ commands:
       - checkout
       - emsdk-env
       - bootstrap
-      - pip-install
       - when:
           # We only set EMTEST_RETRY_FLAKY on pull requests.  When we run
           # normal CI jobs on branches like main we still want to be able to
@@ -871,7 +864,6 @@ jobs:
     executor: linux-python
     steps:
       - checkout
-      - pip-install
       - install-emsdk
       - run:
           name: install jsc
@@ -893,7 +885,6 @@ jobs:
     executor: linux-python
     steps:
       - checkout
-      - pip-install
       - install-emsdk
       - run:
           name: install spidermonkey
@@ -920,7 +911,6 @@ jobs:
       EMTEST_SKIP_V8: "1"
     steps:
       - checkout
-      - pip-install
       - install-emsdk
       # `install-node-version` only changes the NODE_JS_TEST (the version of
       # node used to run test), not NODE_JS (the version of node used to run the
@@ -1078,7 +1068,6 @@ jobs:
       EMTEST_LACKS_WEBGPU: "1"
     steps:
       - checkout
-      - pip-install
       - install-emsdk
       - run-tests-chrome:
           title: "browser"
@@ -1162,7 +1151,6 @@ jobs:
     executor: focal
     steps:
       - checkout
-      - pip-install
       - install-emsdk
       - run-tests-firefox:
           title: "browser64"
@@ -1191,8 +1179,6 @@ jobs:
           name: Add python to bash path
           command: echo "export PATH=\"$PATH:/c/Python27amd64/\"" >> $BASH_ENV
       - install-emsdk
-      - pip-install:
-          python: "$EMSDK_PYTHON"
       - run-tests-firefox-windows:
           title: "browser on firefox on windows"
           # skip browser.test_glbook, as it requires mingw32-make, which is not
@@ -1256,8 +1242,6 @@ jobs:
       # note we do *not* build all libraries and freeze the cache; as we run
       # only limited tests here, it's more efficient to build on demand
       - install-emsdk
-      - pip-install:
-          python: "$EMSDK_PYTHON"
       - run-tests:
           title: "crossplatform tests"
           test_targets: "--crossplatform-only"
@@ -1285,8 +1269,6 @@ jobs:
     steps:
       - setup-macos
       - install-emsdk
-      - pip-install:
-          python: "$EMSDK_PYTHON"
       - freeze-cache
       - run-tests:
           title: "crossplatform tests"

bootstrap.py

@@ -39,6 +39,9 @@
      'test/third_party/googletest',
      'test/third_party/wasi-test-suite',
    ], ['git', 'submodule', 'update', '--init']),
+  ('pip install', [
+     'requirements-dev.txt',
+   ], [sys.executable, '-m', 'pip', 'install', '--root', 'out/python_deps', '-r', 'requirements-dev.txt']),
 ]
 
 
@@ -84,6 +87,10 @@ def main(args):
       shutil.copy(utils.path_from_root(src), dst)
     return 0
 
+  env = os.environ.copy()
+  env['PATH'] += os.pathsep + utils.path_from_root('out/python_deps')
+  env['PYTHONPATH'] = utils.path_from_root('out/python_deps')
+
   for name, deps, cmd in actions:
     if check_deps(name, deps):
       print('Up-to-date: %s' % name)
@@ -99,7 +106,7 @@ def main(args):
       if not cmd[0]:
         utils.exit_with_error(f'command not found: {orig_exe}')
     print(' -> %s' % ' '.join(cmd))
-    subprocess.run(cmd, check=True, text=True, encoding='utf-8', cwd=utils.path_from_root())
+    subprocess.run(cmd, check=True, text=True, encoding='utf-8', cwd=utils.path_from_root(), env=env)
     utils.safe_ensure_dirs(STAMP_DIR)
     utils.write_file(stamp_file, 'Timestamp file created by bootstrap.py')
   return 0

pyproject.toml

@@ -86,6 +86,7 @@ mypy_path = "third_party/,third_party/ply,third_party/websockify"
 files = [ "." ]
 exclude = '''
 (?x)(
+out |
 cache |
 third_party |
 conf\.py |

test/runner.py

@@ -32,11 +32,24 @@
 import unittest
 from functools import cmp_to_key
 
-# Setup
-
 __rootpath__ = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.insert(0, __rootpath__)
 
+# Before we do anthing else, check that the bootstrap script has been run.
+import bootstrap  # noqa: I001
+bootstrap.check()
+
+# Add `out/python_deps` to the python path.  This is where the bootstrap
+# script install our python dev dependencis.
+sys.path.insert(0, os.path.join(__rootpath__, 'out', 'python_deps'))
+# Verify that we can import these dev dependencies and early exit if they
+# are missing
+try:
+  import psutil  # noqa: F401
+  import websockify  # type: ignore # noqa: F401
+except ModuleNotFoundError as e:
+  raise Exception('Unable to import python dev dependencies (psutil/websockify). Run "./bootstrap" (or "python3 -m pip -r requirements-dev.txt --target out/python_deps") to install') from e
+
 import browser_common
 import common
 import jsrun

test/test_sockets.py

@@ -12,6 +12,8 @@
 from subprocess import Popen
 from typing import List
 
+import websockify  # type: ignore
+
 if __name__ == '__main__':
   raise Exception('do not run this file directly; do something like: test/runner sockets')
 
@@ -34,22 +36,9 @@
 
 npm_checked = False
 
-EMTEST_SKIP_PYTHON_DEV_PACKAGES = int(os.getenv('EMTEST_SKIP_PYTHON_DEV_PACKAGES', '0'))
 EMTEST_SKIP_NODE_DEV_PACKAGES = int(os.getenv('EMTEST_SKIP_NODE_DEV_PACKAGES', '0'))
 
 
-def requires_python_dev_packages(func):
-  assert callable(func)
-
-  @common.wraps(func)
-  def decorated(self, *args, **kwargs):
-    if EMTEST_SKIP_PYTHON_DEV_PACKAGES:
-      return self.skipTest('python websockify based tests are disabled by EMTEST_SKIP_PYTHON_DEV_PACKAGES=1')
-    return func(self, *args, **kwargs)
-
-  return decorated
-
-
 def clean_processes(processes):
   for p in processes:
     if getattr(p, 'exitcode', None) is None and getattr(p, 'returncode', None) is None:
@@ -86,11 +75,6 @@ def __enter__(self):
       process = Popen([os.path.abspath('server')])
       self.processes.append(process)
 
-    try:
-      import websockify  # type: ignore # noqa: PLC0415
-    except ModuleNotFoundError:
-      raise Exception('Unable to import module websockify. Run "python3 -m pip install websockify" or set environment variable EMTEST_SKIP_PYTHON_DEV_PACKAGES=1 to skip this test.') from None
-
     # start the websocket proxy
     print('running websockify on %d, forward to tcp %d' % (self.listen_port, self.target_port), file=sys.stderr)
     # source_is_ipv6=True here signals to websockify that it should prefer ipv6 address when
@@ -211,8 +195,6 @@ def setUpClass(cls):
   def test_sockets_echo(self, harness_class, port, args):
     if harness_class == WebsockifyServerHarness and common.EMTEST_LACKS_NATIVE_CLANG:
       self.skipTest('requires native clang')
-    if harness_class == WebsockifyServerHarness and EMTEST_SKIP_PYTHON_DEV_PACKAGES:
-      self.skipTest('requires python websockify and EMTEST_SKIP_PYTHON_DEV_PACKAGES=1')
     if harness_class == CompiledServerHarness and EMTEST_SKIP_NODE_DEV_PACKAGES:
       self.skipTest('requires node ws and EMTEST_SKIP_NODE_DEV_PACKAGES=1')
 
@@ -239,8 +221,6 @@ def test_sdl2_sockets_echo(self):
   def test_sockets_async_echo(self, harness_class, port, args):
     if harness_class == WebsockifyServerHarness and common.EMTEST_LACKS_NATIVE_CLANG:
       self.skipTest('requires native clang')
-    if harness_class == WebsockifyServerHarness and EMTEST_SKIP_PYTHON_DEV_PACKAGES:
-      self.skipTest('requires python websockify and EMTEST_SKIP_PYTHON_DEV_PACKAGES=1')
     if harness_class == CompiledServerHarness and EMTEST_SKIP_NODE_DEV_PACKAGES:
       self.skipTest('requires node ws and EMTEST_SKIP_NODE_DEV_PACKAGES=1')
 
@@ -261,8 +241,6 @@ def test_sockets_async_bad_port(self):
   def test_sockets_echo_bigdata(self, harness_class, port, args):
     if harness_class == WebsockifyServerHarness and common.EMTEST_LACKS_NATIVE_CLANG:
       self.skipTest('requires native clang')
-    if harness_class == WebsockifyServerHarness and EMTEST_SKIP_PYTHON_DEV_PACKAGES:
-      self.skipTest('requires python websockify and EMTEST_SKIP_PYTHON_DEV_PACKAGES=1')
     if harness_class == CompiledServerHarness and EMTEST_SKIP_NODE_DEV_PACKAGES:
       self.skipTest('requires node ws and EMTEST_SKIP_NODE_DEV_PACKAGES=1')
     sockets_include = '-I' + test_file('sockets')
@@ -280,7 +258,6 @@ def test_sockets_echo_bigdata(self, harness_class, port, args):
       self.btest_exit('test_sockets_echo_bigdata.c', cflags=[sockets_include, '-DSOCKK=%d' % harness.listen_port] + args)
 
   @no_windows('This test is Unix-specific.')
-  @requires_python_dev_packages
   @requires_dev_dependency('ws')
   def test_sockets_partial(self):
     for harness in [
@@ -291,7 +268,6 @@ def test_sockets_partial(self):
         self.btest_exit('sockets/test_sockets_partial_client.c', assert_returncode=165, cflags=['-DSOCKK=%d' % harness.listen_port])
 
   @no_windows('This test is Unix-specific.')
-  @requires_python_dev_packages
   @requires_dev_dependency('ws')
   def test_sockets_select_server_down(self):
     for harness in [
@@ -302,7 +278,6 @@ def test_sockets_select_server_down(self):
         self.btest_exit('sockets/test_sockets_select_server_down_client.c', cflags=['-DSOCKK=%d' % harness.listen_port])
 
   @no_windows('This test is Unix-specific.')
-  @requires_python_dev_packages
   @requires_dev_dependency('ws')
   def test_sockets_select_server_closes_connection_rw(self):
     for harness in [
@@ -335,8 +310,6 @@ def test_enet(self):
   def test_nodejs_sockets_echo(self, harness_class, port, args):
     if harness_class == WebsockifyServerHarness and common.EMTEST_LACKS_NATIVE_CLANG:
       self.skipTest('requires native clang')
-    if harness_class == WebsockifyServerHarness and EMTEST_SKIP_PYTHON_DEV_PACKAGES:
-      self.skipTest('requires python websockify and EMTEST_SKIP_PYTHON_DEV_PACKAGES=1')
     if harness_class == CompiledServerHarness and EMTEST_SKIP_NODE_DEV_PACKAGES:
       self.skipTest('requires node ws and EMTEST_SKIP_NODE_DEV_PACKAGES=1')
 
@@ -349,7 +322,6 @@ def test_nodejs_sockets_connect_failure(self):
     self.do_runf('sockets/test_sockets_echo_client.c', r'connect failed: (Connection refused|Host is unreachable)', regex=True, cflags=['-DSOCKK=666'], assert_returncode=NON_ZERO)
 
   @requires_native_clang
-  @requires_python_dev_packages
   def test_nodejs_sockets_echo_subprotocol(self):
     # Test against a Websockified server with compile time configured WebSocket subprotocol. We use a Websockified
     # server because as long as the subprotocol list contains binary it will configure itself to accept binary
@@ -362,7 +334,6 @@ def test_nodejs_sockets_echo_subprotocol(self):
       self.assertContained(['connect: ws://127.0.0.1:59168, base64,binary', 'connect: ws://127.0.0.1:59168/, base64,binary'], out)
 
   @requires_native_clang
-  @requires_python_dev_packages
   def test_nodejs_sockets_echo_subprotocol_runtime(self):
     # Test against a Websockified server with runtime WebSocket configuration. We specify both url and subprotocol.
     # In this test we have *deliberately* used the wrong port '-DSOCKK=12345' to configure the echo_client.c, so