fix video-streamer zombie processes

Author: walesch-yan

docs/conf.py

@@ -14,8 +14,6 @@
 
 # This is necessary to for the `autoflask` directive to get access to the routes
 server, _ = mxcubeweb.build_server_and_config(test=True, argv=[])
-# This is to avoid `Server.kill_processes`, that makes the build return non-zero
-server.flask.testing = True
 
 
 # -- General configuration ------------------------------------------------

mxcubeweb/server.py

@@ -1,7 +1,5 @@
-import atexit
 import logging
 import os
-import signal
 import traceback
 
 import flask_security
@@ -46,23 +44,6 @@ def exception_handler(e):
         logging.getLogger("exceptions").exception(err_msg)
         return err_msg + ": " + traceback.format_exc(), 409
 
-    @staticmethod
-    def kill_processes():
-        # Killing the processes causes pytest to fail because
-        # of non-zero exit code, so we don't kill the processes
-        # when running the tests
-        if not Server.flask.testing:
-            with open("/tmp/mxcube.pid") as f:
-                pid_list = f.read().strip()
-                pid_list = pid_list.split(" ")
-                pid_list.reverse()
-
-            with open("/tmp/mxcube.pid", "w") as f:
-                f.write("")
-
-            for pid in pid_list:
-                os.kill(int(pid), signal.SIGKILL)
-
     @staticmethod
     def init(cmdline_options, cfg):
         template_dir = os.path.join(os.path.dirname(__file__), "templates")
@@ -102,11 +83,6 @@ def init(cmdline_options, cfg):
         # the following test prevents Flask from initializing twice
         # (because of the Reloader)
         if not Server.flask.debug or os.environ.get("WERKZEUG_RUN_MAIN") == "true":
-            atexit.register(Server.kill_processes)
-
-            with open("/tmp/mxcube.pid", "w") as f:
-                f.write(str(os.getpid()) + " ")
-
             # Make the valid_login_only decorator available on server object
             Server.restrict = staticmethod(networkutils.auth_required)
             Server.require_control = staticmethod(networkutils.require_control)

poetry.lock

@@ -49,15 +49,15 @@ pydantic = ">=2.8.2,<2.9.0"
 PyDispatcher = "^2.0.6"
 pytz = "^2022.6"
 tzlocal = "^4.2"
-mxcubecore =  ">=1.198.0"
-mxcube-video-streamer = ">=1.7.0"
+mxcubecore =  ">=1.248.0"
 bcrypt = "^4.0.1"
 authlib = "^1.3.0"
 
 [tool.poetry.group.dev.dependencies]
 pre-commit = "^4.1.0"
 pytest = "^8.3.4"
 pytest-cov = "^6.0.0"
+psutil = "^5.9.4"
 
 [tool.poetry.group.docs.dependencies]
 furo = "^2023.9.10"

pyproject.toml

@@ -549,9 +549,9 @@ importlib-resources==6.4.5 ; python_version >= "3.10" and python_version < "3.12
 itsdangerous==2.2.0 ; python_version >= "3.10" and python_version < "3.12" \
     --hash=sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef \
     --hash=sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173
-jinja2==3.1.5 ; python_version >= "3.10" and python_version < "3.12" \
-    --hash=sha256:8fefff8dc3034e27bb80d67c671eb8a9bc424c0ef4c0826edbff304cceff43bb \
-    --hash=sha256:aba0f4dc9ed8013c424088f68a5c226f7d6097ed89b246d7749c2ec4175c6adb
+jinja2==3.1.6 ; python_version >= "3.10" and python_version < "3.12" \
+    --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \
+    --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67
 jsonpickle==2.2.0 ; python_version >= "3.10" and python_version < "3.12" \
     --hash=sha256:7b272918b0554182e53dc340ddd62d9b7f902fec7e7b05620c04f3ccef479a0e \
     --hash=sha256:de7f2613818aa4f234138ca11243d6359ff83ae528b2185efdd474f62bcf9ae1
@@ -884,41 +884,49 @@ matplotlib==3.7.5 ; python_version >= "3.10" and python_version < "3.12" \
 mock==4.0.3 ; python_version >= "3.10" and python_version < "3.12" \
     --hash=sha256:122fcb64ee37cfad5b3f48d7a7d51875d7031aaf3d8be7c42e2bee25044eee62 \
     --hash=sha256:7d3fbbde18228f4ff2f1f119a45cdffa458b4c0dee32eb4d2bb2f82554bac7bc
-mxcube-video-streamer==1.7.0 ; python_version >= "3.10" and python_version < "3.12" \
-    --hash=sha256:272d0dc033f7b2d9977baa4b7ac2236a4cf208f2324fcbe5832f05cdc140bb35 \
-    --hash=sha256:281d629a3e7171c6ae6045f4ed269e8ab0e2003b50bafc35bdcec761087e8aa2
-mxcubecore==1.208.0 ; python_version >= "3.10" and python_version < "3.12" \
-    --hash=sha256:0e2cc1dd54ab6fab0494d7c7276bcac037b1307e9828261ba1308ab6dcdc663d \
-    --hash=sha256:b5c73626c32b3a9dc606a0f780b41eee2a72c4e819a869d73710a97a727b7a28
-numpy==1.24.4 ; python_version >= "3.10" and python_version < "3.12" \
-    --hash=sha256:04640dab83f7c6c85abf9cd729c5b65f1ebd0ccf9de90b270cd61935eef0197f \
-    --hash=sha256:1452241c290f3e2a312c137a9999cdbf63f78864d63c79039bda65ee86943f61 \
-    --hash=sha256:222e40d0e2548690405b0b3c7b21d1169117391c2e82c378467ef9ab4c8f0da7 \
-    --hash=sha256:2541312fbf09977f3b3ad449c4e5f4bb55d0dbf79226d7724211acc905049400 \
-    --hash=sha256:31f13e25b4e304632a4619d0e0777662c2ffea99fcae2029556b17d8ff958aef \
-    --hash=sha256:4602244f345453db537be5314d3983dbf5834a9701b7723ec28923e2889e0bb2 \
-    --hash=sha256:4979217d7de511a8d57f4b4b5b2b965f707768440c17cb70fbf254c4b225238d \
-    --hash=sha256:4c21decb6ea94057331e111a5bed9a79d335658c27ce2adb580fb4d54f2ad9bc \
-    --hash=sha256:6620c0acd41dbcb368610bb2f4d83145674040025e5536954782467100aa8835 \
-    --hash=sha256:692f2e0f55794943c5bfff12b3f56f99af76f902fc47487bdfe97856de51a706 \
-    --hash=sha256:7215847ce88a85ce39baf9e89070cb860c98fdddacbaa6c0da3ffb31b3350bd5 \
-    --hash=sha256:79fc682a374c4a8ed08b331bef9c5f582585d1048fa6d80bc6c35bc384eee9b4 \
-    --hash=sha256:7ffe43c74893dbf38c2b0a1f5428760a1a9c98285553c89e12d70a96a7f3a4d6 \
-    --hash=sha256:80f5e3a4e498641401868df4208b74581206afbee7cf7b8329daae82676d9463 \
-    --hash=sha256:95f7ac6540e95bc440ad77f56e520da5bf877f87dca58bd095288dce8940532a \
-    --hash=sha256:9667575fb6d13c95f1b36aca12c5ee3356bf001b714fc354eb5465ce1609e62f \
-    --hash=sha256:a5425b114831d1e77e4b5d812b69d11d962e104095a5b9c3b641a218abcc050e \
-    --hash=sha256:b4bea75e47d9586d31e892a7401f76e909712a0fd510f58f5337bea9572c571e \
-    --hash=sha256:b7b1fc9864d7d39e28f41d089bfd6353cb5f27ecd9905348c24187a768c79694 \
-    --hash=sha256:befe2bf740fd8373cf56149a5c23a0f601e82869598d41f8e188a0e9869926f8 \
-    --hash=sha256:c0bfb52d2169d58c1cdb8cc1f16989101639b34c7d3ce60ed70b19c63eba0b64 \
-    --hash=sha256:d11efb4dbecbdf22508d55e48d9c8384db795e1b7b51ea735289ff96613ff74d \
-    --hash=sha256:dd80e219fd4c71fc3699fc1dadac5dcf4fd882bfc6f7ec53d30fa197b8ee22dc \
-    --hash=sha256:e2926dac25b313635e4d6cf4dc4e51c8c0ebfed60b801c799ffc4c32bf3d1254 \
-    --hash=sha256:e98f220aa76ca2a977fe435f5b04d7b3470c0a2e6312907b37ba6068f26787f2 \
-    --hash=sha256:ed094d4f0c177b1b8e7aa9cba7d6ceed51c0e569a5318ac0ca9a090680a6a1b1 \
-    --hash=sha256:f136bab9c2cfd8da131132c2cf6cc27331dd6fae65f95f69dcd4ae3c3639c810 \
-    --hash=sha256:f3a86ed21e4f87050382c7bc96571755193c4c1392490744ac73d660e8f564a9
+mxcube-video-streamer==1.8.2 ; python_version >= "3.10" and python_version < "3.12" \
+    --hash=sha256:0de041c5f17148cec4668a7683f39f1064cd5036d6de311de23a29965466bba6 \
+    --hash=sha256:356998627e4ecc74b6b764dcd4799be0d85e4d658f10810b877d9af14ac57178
+mxcubecore==1.248.0 ; python_version >= "3.10" and python_version < "3.12" \
+    --hash=sha256:24ffe061a8ded03333a6ef7ad7df37767d8bff86f34036e3b27a2d3bdb0e4fcc \
+    --hash=sha256:88b1356f9dbda328e19901c216b70df6797b1c46018445a33fd05daf59fc01db
+numpy==1.26.4 ; python_version >= "3.10" and python_version < "3.12" \
+    --hash=sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b \
+    --hash=sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818 \
+    --hash=sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20 \
+    --hash=sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0 \
+    --hash=sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010 \
+    --hash=sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a \
+    --hash=sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea \
+    --hash=sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c \
+    --hash=sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71 \
+    --hash=sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110 \
+    --hash=sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be \
+    --hash=sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a \
+    --hash=sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a \
+    --hash=sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5 \
+    --hash=sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed \
+    --hash=sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd \
+    --hash=sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c \
+    --hash=sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e \
+    --hash=sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0 \
+    --hash=sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c \
+    --hash=sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a \
+    --hash=sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b \
+    --hash=sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0 \
+    --hash=sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6 \
+    --hash=sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2 \
+    --hash=sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a \
+    --hash=sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30 \
+    --hash=sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218 \
+    --hash=sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5 \
+    --hash=sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07 \
+    --hash=sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2 \
+    --hash=sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4 \
+    --hash=sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764 \
+    --hash=sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef \
+    --hash=sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3 \
+    --hash=sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f
 opencv-python==4.11.0.86 ; python_version >= "3.10" and python_version < "3.12" \
     --hash=sha256:03d60ccae62304860d232272e4a4fda93c39d595780cb40b161b310244b736a4 \
     --hash=sha256:085ad9b77c18853ea66283e98affefe2de8cc4c1f43eda4c100cf9b2721142ec \
@@ -1528,9 +1536,9 @@ tzlocal==4.3.1 ; python_version >= "3.10" and python_version < "3.12" \
 urllib3==2.2.3 ; python_version >= "3.10" and python_version < "3.12" \
     --hash=sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac \
     --hash=sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9
-uvicorn==0.30.6 ; python_version >= "3.10" and python_version < "3.12" \
-    --hash=sha256:4b15decdda1e72be08209e860a1e10e92439ad5b97cf44cc945fcbee66fc5788 \
-    --hash=sha256:65fd46fe3fda5bdc1b03b94eb634923ff18cd35b2f084813ea79d1f103f711b5
+uvicorn==0.34.0 ; python_version >= "3.10" and python_version < "3.12" \
+    --hash=sha256:023dc038422502fa28a09c7a30bf2b6991512da7dcdb8fd35fe57cfc154126f4 \
+    --hash=sha256:404051050cd7e905de2c9a7e61790943440b3416f49cb409f965d9dcd0fa73e9
 websockets==12.0 ; python_version >= "3.10" and python_version < "3.12" \
     --hash=sha256:00700340c6c7ab788f176d118775202aadea7602c5cc6be6ae127761c16d6b0b \
     --hash=sha256:0bee75f400895aef54157b36ed6d3b308fcab62e5260703add87f44cee9c82a6 \

requirements.txt

@@ -4,11 +4,13 @@
 
 monkey.patch_all(thread=False)
 
+import atexit
 import copy
 import json
 import os
 import sys
 
+import psutil
 import pytest
 from input_parameters import (
     test_sample_1,
@@ -33,6 +35,34 @@
 _SIO_TEST_CLIENT = None
 
 
+def noop_register(*args, **kwargs):
+    pass
+
+
+@pytest.fixture(autouse=True)
+def no_atexit_handlers(monkeypatch):
+    # Mock out atexit.register so no handlers are registered during tests
+    monkeypatch.setattr(atexit, "register", noop_register)
+
+
+@pytest.fixture(autouse=True)
+def cleanup_subprocesses():
+    """Fixture to clean up child processes after each test."""
+    parent_pid = os.getpid()
+
+    yield
+
+    parent_process = psutil.Process(parent_pid)
+
+    for child in parent_process.children(recursive=True):
+        if child.is_running():
+            child.terminate()
+            try:
+                child.wait(timeout=1)
+            except psutil.TimeoutExpired:
+                child.kill()
+
+
 @pytest.fixture
 def client():
     with contextlib.suppress(FileNotFoundError):
@@ -43,7 +73,6 @@ def client():
     HardwareRepository.uninit_hardware_repository()
     argv = []
     server, _ = build_server_and_config(test=True, argv=argv)
-    server.flask.config["TESTING"] = True
 
     client = server.flask.test_client()
 

test/conftest.py

@@ -42,7 +42,6 @@ def server(request, login_type):
 
     argv = []
     server_, _ = mxcubeweb.build_server_and_config(test=True, argv=argv)
-    server_.flask.config["TESTING"] = True
     # For the tests we override the configured value of the session lifetime
     # with a much smaller value, so that tests do not need to wait as long.
     server_.flask.permanent_session_lifetime = SESSION_LIFETIME