From bfa50736afa9b32e3f685537498223a743ae6c3a Mon Sep 17 00:00:00 2001 From: Hans Maerki Date: Tue, 21 Oct 2025 11:38:32 +0200 Subject: [PATCH 01/15] UART blocks after ~40 tests --- .gitignore | 1 + .vscode/launch.json | 23 ++++ pyproject.toml | 1 + src/testbed_micropython/mpstress/__init__.py | 1 + src/testbed_micropython/mpstress/cli.py | 123 ++++++++++++++++++ .../mpstress/util_stress.py | 89 +++++++++++++ .../mpstress/util_test_run.py | 53 ++++++++ 7 files changed, 291 insertions(+) create mode 100644 src/testbed_micropython/mpstress/__init__.py create mode 100644 src/testbed_micropython/mpstress/cli.py create mode 100644 src/testbed_micropython/mpstress/util_stress.py create mode 100644 src/testbed_micropython/mpstress/util_test_run.py diff --git a/.gitignore b/.gitignore index e70f445..cc83445 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ src/testbed_micropython/mpbuild/tests/results/*.txt src/testbed_micropython/mpbuild/docker-build-micropython-esp32/esp-idf src/testbed/experiments/* testresults*/** +src/testbed_micropython/mpstress/testresults/ diff --git a/.vscode/launch.json b/.vscode/launch.json index 935d7b5..13caa37 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -213,6 +213,29 @@ "justMyCode": false, "subProcess": false, }, + { + "name": "mpstress stress", + "type": "debugpy", + "request": "launch", + "module": "testbed_micropython.mpstress.cli", + "cwd": "${workspaceFolder}", + "args": [ + // "--help", + // "stress", + "--micropython-tests=${workspaceFolder}/../fork_micropython", + // "--scenario=NONE", + // "--scenario=DUT_ON_OFF", + "--scenario=INFRA_MPREMOTE", + // "--tentacle=1722", + "--tentacle=5f2c", + ], + "console": "integratedTerminal", + "env": { + "PYDEVD_DISABLE_FILE_VALIDATION": "1", + }, + "justMyCode": false, + "subProcess": false, + }, { "name": "pytest", "type": "debugpy", diff --git a/pyproject.toml b/pyproject.toml index 443b84d..1792c82 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ Repository = "https://github.com/octoprobe/octoprobe" [project.scripts] mptest = "testbed_micropython.mptest.cli:app" +mpstress = "testbed_micropython.mpstress.cli:app" [project.optional-dependencies] diff --git a/src/testbed_micropython/mpstress/__init__.py b/src/testbed_micropython/mpstress/__init__.py new file mode 100644 index 0000000..b1a19e3 --- /dev/null +++ b/src/testbed_micropython/mpstress/__init__.py @@ -0,0 +1 @@ +__version__ = "0.0.5" diff --git a/src/testbed_micropython/mpstress/cli.py b/src/testbed_micropython/mpstress/cli.py new file mode 100644 index 0000000..7a85072 --- /dev/null +++ b/src/testbed_micropython/mpstress/cli.py @@ -0,0 +1,123 @@ +""" +Where to take the tests from +--micropython-tests-giturl=https://github.com/dpgeorge/micropython.git@tests-full-test-runner + +Where to take the firmware from +--firmware-build-giturl=https://github.com/micropython/micropython.git@v1.24.1 +--firmware-build-gitdir=~/micropython +--firmware-gitdir=~/micropython +""" + +from __future__ import annotations + +import logging +import pathlib + +import typer +import typing_extensions +from octoprobe.scripts.commissioning import init_logging +from octoprobe.usb_tentacle.usb_tentacle import serial_short_from_delimited + +from testbed_micropython.tentacle_spec import TentacleMicropython +from testbed_micropython.testcollection.baseclasses_spec import ConnectedTentacles + +from .. import constants +from ..mptest import util_testrunner +from .util_stress import EnumScenario, StressThread +from .util_test_run import run_test + +logger = logging.getLogger(__file__) + +# 'typer' does not work correctly with typing.Annotated +# Required is: typing_extensions.Annotated +TyperAnnotated = typing_extensions.Annotated + +# mypy: disable-error-code="valid-type" +# This will disable this warning: +# op.py:58: error: Variable "octoprobe.scripts.op.TyperAnnotated" is not valid as a type [valid-type] +# op.py:58: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases + +app = typer.Typer(pretty_exceptions_enable=False) + +DIRECTORY_OF_THIS_FILE = pathlib.Path(__file__).parent +DIRECTORY_RESULTS = DIRECTORY_OF_THIS_FILE / "testresults" +DIRECTORY_RESULTS.mkdir(parents=True, exist_ok=True) + + +def complete_scenario(): + return sorted([scenario.name for scenario in EnumScenario]) + + +def complete_only_tentacle(): + connected_tentacles = util_testrunner.query_connected_tentacles_fast() + + return sorted( + { + serial_short_from_delimited(t.tentacle_instance.serial) + for t in connected_tentacles + } + ) + + +@app.command(help="Put load on all tentacles to provoke stress") +def stress( + micropython_tests: TyperAnnotated[ + str, + typer.Option( + envvar="TESTBED_MICROPYTHON_MICROPYTHON_TESTS", + help="Directory of MicroPython-Repo with the tests. Example ~/micropython or https://github.com/micropython/micropython.git@v1.24.1", + ), + ] = constants.URL_FILENAME_DEFAULT, + tentacle: TyperAnnotated[ + list[str], + typer.Option( + help="Run tests only on these tentacles. All other tentacles are used to create stress", + autocompletion=complete_only_tentacle, + ), + ] = None, # noqa: UP007 + scenario: TyperAnnotated[ + str, + typer.Option( + help="Run this FUT (feature under test).", autocompletion=complete_scenario + ), + ] = EnumScenario.DUT_ON_OFF, +) -> None: + init_logging() + + try: + connected_tentacles = util_testrunner.query_connected_tentacles_fast() + tentacle_test: TentacleMicropython | None = None + tentacles_load: ConnectedTentacles = ConnectedTentacles() + for t in connected_tentacles: + serial_short = serial_short_from_delimited(t.tentacle_instance.serial) + if serial_short in tentacle: + tentacle_test = t + continue + tentacles_load.append(t) + assert tentacle_test is not None, f"Tentacle not connected: {tentacle}" + for t in connected_tentacles: + t.power.set_default_infra_on() + + repo_micropython_tests = pathlib.Path(micropython_tests).resolve() + assert repo_micropython_tests.is_dir(), repo_micropython_tests + + st = StressThread( + scenario=EnumScenario[scenario], + tentacles_stress=tentacles_load, + directory_results=DIRECTORY_RESULTS, + ) + st.start() + run_test( + tentacle_test=tentacle_test, + repo_micropython_tests=repo_micropython_tests, + directory_results=DIRECTORY_RESULTS, + ) + st.stop() + + except util_testrunner.OctoprobeAppExitException as e: + logger.info(f"Terminating test due to OctoprobeAppExitException: {e}") + raise typer.Exit(1) from e + + +if __name__ == "__main__": + app() diff --git a/src/testbed_micropython/mpstress/util_stress.py b/src/testbed_micropython/mpstress/util_stress.py new file mode 100644 index 0000000..215afec --- /dev/null +++ b/src/testbed_micropython/mpstress/util_stress.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +import enum +import logging +import pathlib +import threading +import time + +from testbed_micropython.testcollection.baseclasses_spec import ConnectedTentacles + +logger = logging.getLogger(__file__) + + +class EnumScenario(enum.StrEnum): + NONE = enum.auto() + DUT_ON_OFF = enum.auto() + INFRA_MPREMOTE = enum.auto() + + +class StressThread(threading.Thread): + def __init__( + self, + scenario: EnumScenario, + tentacles_stress: ConnectedTentacles, + directory_results: pathlib.Path, + ): + assert isinstance(scenario, EnumScenario) + assert isinstance(tentacles_stress, ConnectedTentacles) + assert isinstance(directory_results, pathlib.Path) + super().__init__(daemon=True, name="stress") + self._stopping = False + self._scenario = scenario + self._tentacles_stress = tentacles_stress + self._directory_results = directory_results + + def run(self) -> None: + """ + Power up all duts on all tentacles. + Now loop over all tentacles and power down dut for a short time + """ + print(f"Found {len(self._tentacles_stress)} tentacle to create stress.") + + if self._scenario is EnumScenario.NONE: + return self._scenario_NONE() + + if self._scenario is EnumScenario.DUT_ON_OFF: + return self._scenario_DUT_ON_OFF() + + if self._scenario is EnumScenario.INFRA_MPREMOTE: + return self._scenario_INFRA_MPREMOTE() + + assert False + + def _scenario_NONE(self) -> None: + return + + def _scenario_INFRA_MPREMOTE(self) -> None: + print("on") + for t in self._tentacles_stress: + t.infra.power.dut = True + + while True: + if self._stopping: + return + print("cycle") + for t in self._tentacles_stress: + t.infra.mp_remote_close() + t.infra.connect_mpremote_if_needed() + print(t.infra.mp_remote._tty) + rc = t.infra.mp_remote.exec_raw("print('Hello')") + assert rc == "Hello\r\n" + # print(rc) + + def _scenario_DUT_ON_OFF(self) -> None: + sleep_s = 1.0 + while not self._stopping: + print("on") + for t in self._tentacles_stress: + t.infra.power.dut = True + time.sleep(sleep_s) + + print("off") + for t in self._tentacles_stress: + t.infra.power.dut = False + time.sleep(sleep_s) + + def stop(self) -> None: + self._stopping = True + self.join() diff --git a/src/testbed_micropython/mpstress/util_test_run.py b/src/testbed_micropython/mpstress/util_test_run.py new file mode 100644 index 0000000..c256c02 --- /dev/null +++ b/src/testbed_micropython/mpstress/util_test_run.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import logging +import pathlib +import sys + +from octoprobe.util_pyudev import UdevPoller +from octoprobe.util_subprocess import subprocess_run + +from testbed_micropython.tentacle_spec import TentacleMicropython + +from ..testcollection.constants import ( + ENV_PYTHONUNBUFFERED, + MICROPYTHON_DIRECTORY_TESTS, +) + +logger = logging.getLogger(__file__) + + +def run_test( + tentacle_test: TentacleMicropython, + repo_micropython_tests: pathlib.Path, + directory_results: pathlib.Path, +) -> None: + assert isinstance(tentacle_test, TentacleMicropython) + assert isinstance(repo_micropython_tests, pathlib.Path) + assert isinstance(directory_results, pathlib.Path) + + # tentacle_test.power.dut = True + with UdevPoller() as udev: + tty = tentacle_test.dut.dut_mcu.application_mode_power_up( + tentacle=tentacle_test, udev=udev + ) + + args = [ + sys.executable, + "run-tests.py", + f"--result-dir={directory_results}", + f"--test-instance=port:{tty}", + "--jobs=1", + # "misc/cexample_class.py", + ] + env = ENV_PYTHONUNBUFFERED + subprocess_run( + args=args, + cwd=repo_micropython_tests / MICROPYTHON_DIRECTORY_TESTS, + env=env, + # logfile=testresults_directory(f"run-tests-{test_dir}.txt").filename, + logfile=directory_results / "testresults.txt", + timeout_s=300.0, + # TODO: Remove the following line as soon returncode of 'run-multitest.py' is fixed. + success_returncodes=[0, 1], + ) From 1b3475390c85da8875ba88a3b0bb80fbb8a90aec Mon Sep 17 00:00:00 2001 From: Hans Maerki Date: Tue, 21 Oct 2025 15:17:34 +0200 Subject: [PATCH 02/15] UART blocks after basics/builtin_pow3.py --- .vscode/launch.json | 2 +- .../mpstress/util_test_run.py | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 13caa37..f544280 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -214,7 +214,7 @@ "subProcess": false, }, { - "name": "mpstress stress", + "name": "mpstress", "type": "debugpy", "request": "launch", "module": "testbed_micropython.mpstress.cli", diff --git a/src/testbed_micropython/mpstress/util_test_run.py b/src/testbed_micropython/mpstress/util_test_run.py index c256c02..6f6f8ca 100644 --- a/src/testbed_micropython/mpstress/util_test_run.py +++ b/src/testbed_micropython/mpstress/util_test_run.py @@ -32,12 +32,29 @@ def run_test( tentacle=tentacle_test, udev=udev ) + if True: + timeout_s = 240.0 * 1.5 + files = [] + if True: + timeout_s = 61.0 * 1.5 + files = ["--include=basic/"] + if True: + timeout_s = 77.0 * 1.5 + files = ["--include=extmod/"] + if True: + timeout_s = 17.0 * 1.5 + files = ["--include=basics/b"] + if True: + timeout_s = 22.0 * 1.5 + files = ["--include=basics/(b|int_)"] + args = [ sys.executable, "run-tests.py", f"--result-dir={directory_results}", f"--test-instance=port:{tty}", "--jobs=1", + *files, # "misc/cexample_class.py", ] env = ENV_PYTHONUNBUFFERED @@ -47,7 +64,7 @@ def run_test( env=env, # logfile=testresults_directory(f"run-tests-{test_dir}.txt").filename, logfile=directory_results / "testresults.txt", - timeout_s=300.0, + timeout_s=timeout_s, # TODO: Remove the following line as soon returncode of 'run-multitest.py' is fixed. success_returncodes=[0, 1], ) From 111f02fb6e35413634df03460909e88401f6cafb Mon Sep 17 00:00:00 2001 From: Hans Maerki Date: Tue, 21 Oct 2025 18:34:56 +0200 Subject: [PATCH 03/15] Introduced scenarios --- .vscode/launch.json | 1 + src/testbed_micropython/mpstress/cli.py | 2 + .../mpstress/util_stress.py | 91 +++++++++++++++++-- .../mpstress/util_test_run.py | 16 +++- 4 files changed, 99 insertions(+), 11 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index f544280..03fec68 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -226,6 +226,7 @@ // "--scenario=NONE", // "--scenario=DUT_ON_OFF", "--scenario=INFRA_MPREMOTE", + // "--scenario=SUBPROCESS_INFRA_MPREMOTE", // "--tentacle=1722", "--tentacle=5f2c", ], diff --git a/src/testbed_micropython/mpstress/cli.py b/src/testbed_micropython/mpstress/cli.py index 7a85072..dc4bd54 100644 --- a/src/testbed_micropython/mpstress/cli.py +++ b/src/testbed_micropython/mpstress/cli.py @@ -42,6 +42,8 @@ DIRECTORY_OF_THIS_FILE = pathlib.Path(__file__).parent DIRECTORY_RESULTS = DIRECTORY_OF_THIS_FILE / "testresults" DIRECTORY_RESULTS.mkdir(parents=True, exist_ok=True) +[f.unlink(missing_ok=True) for f in DIRECTORY_RESULTS.glob("*.txt")] +[f.unlink(missing_ok=True) for f in DIRECTORY_RESULTS.glob("*.out")] def complete_scenario(): diff --git a/src/testbed_micropython/mpstress/util_stress.py b/src/testbed_micropython/mpstress/util_stress.py index 215afec..28657e8 100644 --- a/src/testbed_micropython/mpstress/util_stress.py +++ b/src/testbed_micropython/mpstress/util_stress.py @@ -3,18 +3,53 @@ import enum import logging import pathlib +import sys import threading import time +from octoprobe.util_subprocess import subprocess_run + from testbed_micropython.testcollection.baseclasses_spec import ConnectedTentacles +from ..testcollection.constants import ( + ENV_PYTHONUNBUFFERED, +) + logger = logging.getLogger(__file__) class EnumScenario(enum.StrEnum): NONE = enum.auto() DUT_ON_OFF = enum.auto() + """ + 13 tentacles + + timeout_s = 240.0 * 1.5 + files = ["--exclude=ports/rp2/rp2_lightsleep_thread.py"] # Broken test + + --> no error! + """ INFRA_MPREMOTE = enum.auto() + """ + 13 tentacles + + timeout_s = 240.0 * 1.5 + files = [] + + --> no error! + """ + SUBPROCESS_INFRA_MPREMOTE = enum.auto() + """ + 13 tentacles + + timeout_s = 13.0 * 1.5 + files = [ + "--include=basics/(b|int_)", + "--exclude=basics/builtin_pow", + ] + + --> error after 20s + """ class StressThread(threading.Thread): @@ -48,25 +83,67 @@ def run(self) -> None: if self._scenario is EnumScenario.INFRA_MPREMOTE: return self._scenario_INFRA_MPREMOTE() + if self._scenario is EnumScenario.SUBPROCESS_INFRA_MPREMOTE: + return self._scenario_SUBPROCESS_INFRA_MPREMOTE() assert False def _scenario_NONE(self) -> None: return + def _scenario_SUBPROCESS_INFRA_MPREMOTE(self) -> None: + i = 0 + while True: + print("cycle") + for t in self._tentacles_stress: + if self._stopping: + return + i += 1 + args = [ + sys.executable, + "-m", + "mpremote", + "connect", + t.infra.usb_tentacle.serial_port, + "eval", + "print('Hello MicroPython')", + ] + env = ENV_PYTHONUNBUFFERED + subprocess_run( + args=args, + cwd=self._directory_results, + env=env, + # logfile=testresults_directory(f"run-tests-{test_dir}.txt").filename, + logfile=self._directory_results / f"mpremote_{i:03d}.txt", + timeout_s=5.0, + ) + def _scenario_INFRA_MPREMOTE(self) -> None: - print("on") + print("off") for t in self._tentacles_stress: - t.infra.power.dut = True + t.infra.power.dut = False + i = 0 while True: - if self._stopping: - return print("cycle") - for t in self._tentacles_stress: - t.infra.mp_remote_close() + for idx, t in enumerate(self._tentacles_stress): + if self._stopping: + return + i += 1 + # if idx > 5: + # continue + serial_closed = t.infra.mp_remote_close() t.infra.connect_mpremote_if_needed() - print(t.infra.mp_remote._tty) + print( + i, + t.infra._mp_remote.state.transport.serial.fd, + t.infra._mp_remote.state.transport.serial.pipe_abort_read_r, + t.infra._mp_remote.state.transport.serial.pipe_abort_read_w, + t.infra._mp_remote.state.transport.serial.pipe_abort_write_r, + t.infra._mp_remote.state.transport.serial.pipe_abort_write_w, + end="", + ) + print(" ", serial_closed, t.infra.mp_remote._tty) rc = t.infra.mp_remote.exec_raw("print('Hello')") assert rc == "Hello\r\n" # print(rc) diff --git a/src/testbed_micropython/mpstress/util_test_run.py b/src/testbed_micropython/mpstress/util_test_run.py index 6f6f8ca..b62c093 100644 --- a/src/testbed_micropython/mpstress/util_test_run.py +++ b/src/testbed_micropython/mpstress/util_test_run.py @@ -34,19 +34,27 @@ def run_test( if True: timeout_s = 240.0 * 1.5 - files = [] + files = ["--exclude=ports/rp2/rp2_lightsleep_thread.py"] # Broken test if True: timeout_s = 61.0 * 1.5 - files = ["--include=basic/"] + files = ["--include=basics/*"] if True: timeout_s = 77.0 * 1.5 - files = ["--include=extmod/"] + files = ["--include=extmod/*"] if True: timeout_s = 17.0 * 1.5 files = ["--include=basics/b"] if True: timeout_s = 22.0 * 1.5 - files = ["--include=basics/(b|int_)"] + files = [ + "--include=basics/(b|int_)", + ] + if True: + timeout_s = 13.0 * 1.5 + files = [ + "--include=basics/(b|int_)", + "--exclude=basics/builtin_pow", + ] args = [ sys.executable, From 80ac12f3f7482e183a3435ea03f43463cc8228ea Mon Sep 17 00:00:00 2001 From: Hans Maerki Date: Tue, 21 Oct 2025 20:49:32 +0200 Subject: [PATCH 04/15] Add scenario SUBPROCESS_INFRA_MPREMOTE_C --- .gitignore | 1 + .vscode/launch.json | 1 + src/testbed_micropython/mpstress/c/Makefile | 12 ++ .../mpstress/c/mpremote_c.c | 147 ++++++++++++++++++ .../mpstress/util_stress.py | 63 ++++++-- 5 files changed, 215 insertions(+), 9 deletions(-) create mode 100644 src/testbed_micropython/mpstress/c/Makefile create mode 100644 src/testbed_micropython/mpstress/c/mpremote_c.c diff --git a/.gitignore b/.gitignore index cc83445..8451565 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ src/testbed_micropython/mpbuild/docker-build-micropython-esp32/esp-idf src/testbed/experiments/* testresults*/** src/testbed_micropython/mpstress/testresults/ +src/testbed_micropython/mpstress/c/mpremote_c diff --git a/.vscode/launch.json b/.vscode/launch.json index 03fec68..4630a30 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -227,6 +227,7 @@ // "--scenario=DUT_ON_OFF", "--scenario=INFRA_MPREMOTE", // "--scenario=SUBPROCESS_INFRA_MPREMOTE", + // "--scenario=SUBPROCESS_INFRA_MPREMOTE_C", // "--tentacle=1722", "--tentacle=5f2c", ], diff --git a/src/testbed_micropython/mpstress/c/Makefile b/src/testbed_micropython/mpstress/c/Makefile new file mode 100644 index 0000000..a0190df --- /dev/null +++ b/src/testbed_micropython/mpstress/c/Makefile @@ -0,0 +1,12 @@ +CC=gcc +CFLAGS=-Wall -Wextra -std=c99 +TARGET=mpremote_c +SOURCE=mpremote_c.c + +$(TARGET): $(SOURCE) + $(CC) $(CFLAGS) -o $(TARGET) $(SOURCE) + +clean: + rm -f $(TARGET) + +.PHONY: clean \ No newline at end of file diff --git a/src/testbed_micropython/mpstress/c/mpremote_c.c b/src/testbed_micropython/mpstress/c/mpremote_c.c new file mode 100644 index 0000000..4871a51 --- /dev/null +++ b/src/testbed_micropython/mpstress/c/mpremote_c.c @@ -0,0 +1,147 @@ +/* +This programs open a serial line to a micropython cpu. + +Send "print('hello')" +Return 0 if the response in "hellon\r\n" +Return 1 on error. +*/ +#define _DEFAULT_SOURCE +#define _BSD_SOURCE +#include +#include +#include +#include +#include +#include +#include + +#ifndef CRTSCTS +#define CRTSCTS 0 +#endif + +int main(int argc, char *argv[]) { + if (argc != 2) { + fprintf(stderr, "Usage: %s \n", argv[0]); + fprintf(stderr, "Example: %s /dev/ttyACM4\n", argv[0]); + return 1; + } + + const char *serial_device = argv[1]; + int serial_fd; + struct termios tty; + const char *command = "print('hello')\r\n"; + char response[256]; + const char *expected_response = "hello\r\n"; + ssize_t bytes_read; + + // Open serial device + serial_fd = open(serial_device, O_RDWR | O_NOCTTY); + if (serial_fd < 0) { + perror("Error opening serial device"); + return 1; + } + + // Get current terminal attributes + if (tcgetattr(serial_fd, &tty) != 0) { + perror("Error getting terminal attributes"); + close(serial_fd); + return 1; + } + + // Configure serial port settings + // Set baud rate to 115200 + cfsetospeed(&tty, B115200); + cfsetispeed(&tty, B115200); + + // 8N1 mode + tty.c_cflag &= ~PARENB; // No parity + tty.c_cflag &= ~CSTOPB; // One stop bit + tty.c_cflag &= ~CSIZE; // Clear size bits + tty.c_cflag |= CS8; // 8 data bits + tty.c_cflag &= ~CRTSCTS; // No hardware flow control + tty.c_cflag |= CREAD | CLOCAL; // Enable reading, ignore modem control lines + + // Input modes + tty.c_iflag &= ~(IXON | IXOFF | IXANY); // No software flow control + tty.c_iflag &= ~(ICANON | ECHO | ECHOE | ISIG); // Raw input + + // Output modes + tty.c_oflag &= ~OPOST; // Raw output + + // Local modes + tty.c_lflag &= ~(ICANON | ECHO | ECHOE | ISIG); // Raw mode + + // Control characters + tty.c_cc[VMIN] = 2*strlen(expected_response); // Blocking read - wait for at least 1 character + tty.c_cc[VTIME] = 10; // 1 second timeout (in deciseconds) + + // Apply settings + if (tcsetattr(serial_fd, TCSANOW, &tty) != 0) { + perror("Error setting terminal attributes"); + close(serial_fd); + return 1; + } + + // Flush any existing data + tcflush(serial_fd, TCIOFLUSH); + + // Give device time to initialize + // usleep(100000); // 100ms + + // Send command + ssize_t bytes_written = write(serial_fd, command, strlen(command)); + if (bytes_written < 0) { + perror("Error writing to serial device"); + close(serial_fd); + return 1; + } + + // Give device time to process and respond + // usleep(500000); // 500ms + + // Read response + memset(response, 0, sizeof(response)); + bytes_read = read(serial_fd, response, sizeof(response) - 1); + if (bytes_read < 0) { + perror("Error reading from serial device"); + close(serial_fd); + return 1; + } + + // Close serial device + close(serial_fd); + + // Check if we got the expected response + // Look for "hello" followed by newline and carriage return + if (strstr(response, "hello") != NULL) { + // Check if response contains hello followed by \r\n or \n\r + if (strstr(response, "hello\r\n") != NULL || strstr(response, "hello\n\r") != NULL) { + printf("Success: Got expected response containing 'hello\\r\\n' or 'hello\\n\\r'\n"); + return 0; + } else if (strstr(response, "hello") != NULL) { + printf("Partial success: Got 'hello' but not with expected line endings\n"); + printf("Response was: "); + for (int i = 0; i < bytes_read; i++) { + if (response[i] >= 32 && response[i] <= 126) { + printf("%c", response[i]); + } else { + printf("\\x%02x", (unsigned char)response[i]); + } + } + printf("\n"); + return 0; // Still consider it success if we got hello + } + } + + printf("Failed: Did not get expected response 'hello\\n\\r'\n"); + printf("Response was: "); + for (int i = 0; i < bytes_read; i++) { + if (response[i] >= 32 && response[i] <= 126) { + printf("%c", response[i]); + } else { + printf("\\x%02x", (unsigned char)response[i]); + } + } + printf("\n"); + return 1; +} diff --git a/src/testbed_micropython/mpstress/util_stress.py b/src/testbed_micropython/mpstress/util_stress.py index 28657e8..6d80919 100644 --- a/src/testbed_micropython/mpstress/util_stress.py +++ b/src/testbed_micropython/mpstress/util_stress.py @@ -15,6 +15,8 @@ ENV_PYTHONUNBUFFERED, ) +DIRECTORY_OF_THIS_FILE = pathlib.Path(__file__).parent + logger = logging.getLogger(__file__) @@ -33,8 +35,20 @@ class EnumScenario(enum.StrEnum): """ 13 tentacles + timeout_s = 13.0 * 1.5 + files = [ + "--include=basics/(b|int_)", + "--exclude=basics/builtin_pow", + ] + + --> error after 20s + + ------------------ + + 5 tentacles + timeout_s = 240.0 * 1.5 - files = [] + files = ["--exclude=ports/rp2/rp2_lightsleep_thread.py"] # Broken test --> no error! """ @@ -42,13 +56,20 @@ class EnumScenario(enum.StrEnum): """ 13 tentacles - timeout_s = 13.0 * 1.5 - files = [ - "--include=basics/(b|int_)", - "--exclude=basics/builtin_pow", - ] + timeout_s = 240.0 * 1.5 + files = ["--exclude=ports/rp2/rp2_lightsleep_thread.py"] # Broken test - --> error after 20s + --> no error! + """ + SUBPROCESS_INFRA_MPREMOTE_C = enum.auto() + """ + 13 tentacles + + timeout_s = 240.0 * 1.5 + files = ["--exclude=ports/rp2/rp2_lightsleep_thread.py"] # Broken test + + mpremote_c was called 3147 times! + --> no error! """ @@ -83,14 +104,39 @@ def run(self) -> None: if self._scenario is EnumScenario.INFRA_MPREMOTE: return self._scenario_INFRA_MPREMOTE() + if self._scenario is EnumScenario.SUBPROCESS_INFRA_MPREMOTE: return self._scenario_SUBPROCESS_INFRA_MPREMOTE() + if self._scenario is EnumScenario.SUBPROCESS_INFRA_MPREMOTE_C: + return self._scenario_SUBPROCESS_INFRA_MPREMOTE_C() + assert False def _scenario_NONE(self) -> None: return + def _scenario_SUBPROCESS_INFRA_MPREMOTE_C(self) -> None: + i = 0 + while True: + print("cycle") + for t in self._tentacles_stress: + if self._stopping: + return + i += 1 + args = [ + str(DIRECTORY_OF_THIS_FILE / "c" / "mpremote_c"), + t.infra.usb_tentacle.serial_port, + ] + env = ENV_PYTHONUNBUFFERED + subprocess_run( + args=args, + cwd=self._directory_results, + env=env, + logfile=self._directory_results / f"mpremote_c_{i:04d}.txt", + timeout_s=1.0, + ) + def _scenario_SUBPROCESS_INFRA_MPREMOTE(self) -> None: i = 0 while True: @@ -113,8 +159,7 @@ def _scenario_SUBPROCESS_INFRA_MPREMOTE(self) -> None: args=args, cwd=self._directory_results, env=env, - # logfile=testresults_directory(f"run-tests-{test_dir}.txt").filename, - logfile=self._directory_results / f"mpremote_{i:03d}.txt", + logfile=self._directory_results / f"mpremote_{i:04d}.txt", timeout_s=5.0, ) From 6d14f8e3a5ff9427c0e740a406da47708b2c7cc5 Mon Sep 17 00:00:00 2001 From: Hans Maerki Date: Wed, 22 Oct 2025 09:16:29 +0200 Subject: [PATCH 05/15] Introduce EnumTest --- .vscode/launch.json | 1 + src/testbed_micropython/mpstress/cli.py | 13 ++- .../mpstress/util_test_run.py | 86 +++++++++++++------ 3 files changed, 73 insertions(+), 27 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 4630a30..d12f43e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -228,6 +228,7 @@ "--scenario=INFRA_MPREMOTE", // "--scenario=SUBPROCESS_INFRA_MPREMOTE", // "--scenario=SUBPROCESS_INFRA_MPREMOTE_C", + "--test=RUN_TESTS_BASIC_B_INT_POW", // "--tentacle=1722", "--tentacle=5f2c", ], diff --git a/src/testbed_micropython/mpstress/cli.py b/src/testbed_micropython/mpstress/cli.py index dc4bd54..4494d5d 100644 --- a/src/testbed_micropython/mpstress/cli.py +++ b/src/testbed_micropython/mpstress/cli.py @@ -24,7 +24,7 @@ from .. import constants from ..mptest import util_testrunner from .util_stress import EnumScenario, StressThread -from .util_test_run import run_test +from .util_test_run import run_test, EnumTest logger = logging.getLogger(__file__) @@ -50,6 +50,10 @@ def complete_scenario(): return sorted([scenario.name for scenario in EnumScenario]) +def complete_test(): + return sorted([test.name for test in EnumTest]) + + def complete_only_tentacle(): connected_tentacles = util_testrunner.query_connected_tentacles_fast() @@ -80,9 +84,13 @@ def stress( scenario: TyperAnnotated[ str, typer.Option( - help="Run this FUT (feature under test).", autocompletion=complete_scenario + help="Run this stress scenario.", autocompletion=complete_scenario ), ] = EnumScenario.DUT_ON_OFF, + test: TyperAnnotated[ + str, + typer.Option(help="Use these test arguments.", autocompletion=complete_test), + ] = EnumTest.RUN_TESTS_BASIC_B_INT_POW, ) -> None: init_logging() @@ -113,6 +121,7 @@ def stress( tentacle_test=tentacle_test, repo_micropython_tests=repo_micropython_tests, directory_results=DIRECTORY_RESULTS, + test=EnumTest[test], ) st.stop() diff --git a/src/testbed_micropython/mpstress/util_test_run.py b/src/testbed_micropython/mpstress/util_test_run.py index b62c093..6a91e50 100644 --- a/src/testbed_micropython/mpstress/util_test_run.py +++ b/src/testbed_micropython/mpstress/util_test_run.py @@ -1,5 +1,7 @@ from __future__ import annotations +import dataclasses +import enum import logging import pathlib import sys @@ -17,14 +19,70 @@ logger = logging.getLogger(__file__) +class EnumTest(enum.StrEnum): + RUN_TESTS_ALL = enum.auto() + RUN_TESTS_BASIC = enum.auto() + RUN_TESTS_BASIC_B = enum.auto() + RUN_TESTS_BASIC_B_INT = enum.auto() + RUN_TESTS_BASIC_B_INT_POW = enum.auto() + RUN_TESTS_EXTMOD = enum.auto() + + @property + def test_params(self) -> TestArgs: + if self is EnumTest.RUN_TESTS_ALL: + return TestArgs( + timeout_s=240.0 * 1.5, + files=["--exclude=ports/rp2/rp2_lightsleep_thread.py"], # Broken test + ) + if self is EnumTest.RUN_TESTS_BASIC: + return TestArgs( + timeout_s=61.0 * 1.5, + files=["--include=basics/*"], + ) + if self is EnumTest.RUN_TESTS_EXTMOD: + return TestArgs( + timeout_s=77.0 * 1.5, + files=["--include=extmod/*"], + ) + if self is EnumTest.RUN_TESTS_BASIC_B: + return TestArgs( + timeout_s=17.0 * 1.5, + files=["--include=basics/b"], + ) + if self is EnumTest.RUN_TESTS_BASIC_B_INT: + return TestArgs( + timeout_s=22.0 * 1.5, + files=[ + "--include=basics/(b|int_)", + ], + ) + if self is EnumTest.RUN_TESTS_BASIC_B_INT_POW: + return TestArgs( + timeout_s=13.0 * 1.5, + files=[ + "--include=basics/(b|int_)", + "--exclude=basics/builtin_pow", + ], + ) + raise ValueError(self) + + +@dataclasses.dataclass(repr=True, frozen=True) +class TestArgs: + timeout_s: float + files: list[str] + + def run_test( tentacle_test: TentacleMicropython, repo_micropython_tests: pathlib.Path, directory_results: pathlib.Path, + test: EnumTest, ) -> None: assert isinstance(tentacle_test, TentacleMicropython) assert isinstance(repo_micropython_tests, pathlib.Path) assert isinstance(directory_results, pathlib.Path) + assert isinstance(test, EnumTest) # tentacle_test.power.dut = True with UdevPoller() as udev: @@ -32,29 +90,7 @@ def run_test( tentacle=tentacle_test, udev=udev ) - if True: - timeout_s = 240.0 * 1.5 - files = ["--exclude=ports/rp2/rp2_lightsleep_thread.py"] # Broken test - if True: - timeout_s = 61.0 * 1.5 - files = ["--include=basics/*"] - if True: - timeout_s = 77.0 * 1.5 - files = ["--include=extmod/*"] - if True: - timeout_s = 17.0 * 1.5 - files = ["--include=basics/b"] - if True: - timeout_s = 22.0 * 1.5 - files = [ - "--include=basics/(b|int_)", - ] - if True: - timeout_s = 13.0 * 1.5 - files = [ - "--include=basics/(b|int_)", - "--exclude=basics/builtin_pow", - ] + test_params = test.test_params args = [ sys.executable, @@ -62,7 +98,7 @@ def run_test( f"--result-dir={directory_results}", f"--test-instance=port:{tty}", "--jobs=1", - *files, + *test_params.files, # "misc/cexample_class.py", ] env = ENV_PYTHONUNBUFFERED @@ -72,7 +108,7 @@ def run_test( env=env, # logfile=testresults_directory(f"run-tests-{test_dir}.txt").filename, logfile=directory_results / "testresults.txt", - timeout_s=timeout_s, + timeout_s=test_params.timeout_s, # TODO: Remove the following line as soon returncode of 'run-multitest.py' is fixed. success_returncodes=[0, 1], ) From a6c84c8334ff1521d5b7292c2882ff2f3daa5e97 Mon Sep 17 00:00:00 2001 From: Hans Maerki Date: Wed, 22 Oct 2025 11:53:12 +0200 Subject: [PATCH 06/15] Add SERIAL_TEST --- .vscode/launch.json | 7 +- .../mpstress/README_flakyness.md | 51 +++++++++++++ src/testbed_micropython/mpstress/cli.py | 9 ++- .../mpstress/util_stress.py | 74 +++++-------------- .../mpstress/util_test_run.py | 41 ++++++++-- 5 files changed, 116 insertions(+), 66 deletions(-) create mode 100644 src/testbed_micropython/mpstress/README_flakyness.md diff --git a/.vscode/launch.json b/.vscode/launch.json index d12f43e..d9cf19b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -222,13 +222,16 @@ "args": [ // "--help", // "stress", - "--micropython-tests=${workspaceFolder}/../fork_micropython", + // "--micropython-tests=${workspaceFolder}/../fork_micropython", + "--micropython-tests=${workspaceFolder}/../micropython", // "--scenario=NONE", // "--scenario=DUT_ON_OFF", "--scenario=INFRA_MPREMOTE", // "--scenario=SUBPROCESS_INFRA_MPREMOTE", // "--scenario=SUBPROCESS_INFRA_MPREMOTE_C", - "--test=RUN_TESTS_BASIC_B_INT_POW", + // "--test=RUN_TESTS_BASIC_B_INT_POW", + "--test=SERIAL_TEST", + "--stress-tentacle-count=10", // "--tentacle=1722", "--tentacle=5f2c", ], diff --git a/src/testbed_micropython/mpstress/README_flakyness.md b/src/testbed_micropython/mpstress/README_flakyness.md new file mode 100644 index 0000000..d47fd26 --- /dev/null +++ b/src/testbed_micropython/mpstress/README_flakyness.md @@ -0,0 +1,51 @@ +# Testresults + +### --scenario=NONE --test=RUN_TESTS_ALL + +12 tentacles +--> no error! + +### --scenario=DUT_ON_OFF --test=RUN_TESTS_ALL + +12 tentacles +--> no error! + +### --scenario=INFRA_MPREMOTE --test=RUN_TESTS_BASIC_B_INT_POW + +12 tentacles +--> error after 20s - sometimes + +### --scenario=INFRA_MPREMOTE --test=RUN_TESTS_BASIC_B_INT_POW --stress-tentacle-count=5 + +5 tentacles +--> no error! + +### --scenario=SUBPROCESS_INFRA_MPREMOTE --test=RUN_TESTS_ALL + +12 tentacles +--> no error! + +### --scenario=SUBPROCESS_INFRA_MPREMOTE_C --test=RUN_TESTS_ALL + +12 tentacles +--> no error! + +### --scenario=INFRA_MPREMOTE --test=SERIAL_TEST + +12 tentacles +--> error after 3s + +### --scenario=INFRA_MPREMOTE --test=SERIAL_TEST --stress-tentacle-count=10 + +10 tentacles +--> error after 1.5s, 2s, 8s + +### --scenario=INFRA_MPREMOTE --test=SERIAL_TEST --stress-tentacle-count=7 + +7 tentacles +--> error after 8s, 14s, 26s + +### --scenario=INFRA_MPREMOTE --test=SERIAL_TEST --stress-tentacle-count=6 + +6 tentacles +--> no error! diff --git a/src/testbed_micropython/mpstress/cli.py b/src/testbed_micropython/mpstress/cli.py index 4494d5d..d8b19b2 100644 --- a/src/testbed_micropython/mpstress/cli.py +++ b/src/testbed_micropython/mpstress/cli.py @@ -24,7 +24,7 @@ from .. import constants from ..mptest import util_testrunner from .util_stress import EnumScenario, StressThread -from .util_test_run import run_test, EnumTest +from .util_test_run import EnumTest, run_test logger = logging.getLogger(__file__) @@ -81,6 +81,12 @@ def stress( autocompletion=complete_only_tentacle, ), ] = None, # noqa: UP007 + stress_tentacle_count: TyperAnnotated[ + int, + typer.Option( + help="Use that many tentacles to generate stress. May be less if less tentacles are connected.", + ), + ] = 99, # noqa: UP007 scenario: TyperAnnotated[ str, typer.Option( @@ -113,6 +119,7 @@ def stress( st = StressThread( scenario=EnumScenario[scenario], + stress_tentacle_count=stress_tentacle_count, tentacles_stress=tentacles_load, directory_results=DIRECTORY_RESULTS, ) diff --git a/src/testbed_micropython/mpstress/util_stress.py b/src/testbed_micropython/mpstress/util_stress.py index 6d80919..b295c08 100644 --- a/src/testbed_micropython/mpstress/util_stress.py +++ b/src/testbed_micropython/mpstress/util_stress.py @@ -7,13 +7,12 @@ import threading import time +from octoprobe.usb_tentacle.usb_tentacle import serial_short_from_delimited from octoprobe.util_subprocess import subprocess_run from testbed_micropython.testcollection.baseclasses_spec import ConnectedTentacles -from ..testcollection.constants import ( - ENV_PYTHONUNBUFFERED, -) +from ..testcollection.constants import ENV_PYTHONUNBUFFERED DIRECTORY_OF_THIS_FILE = pathlib.Path(__file__).parent @@ -23,79 +22,41 @@ class EnumScenario(enum.StrEnum): NONE = enum.auto() DUT_ON_OFF = enum.auto() - """ - 13 tentacles - - timeout_s = 240.0 * 1.5 - files = ["--exclude=ports/rp2/rp2_lightsleep_thread.py"] # Broken test - - --> no error! - """ INFRA_MPREMOTE = enum.auto() - """ - 13 tentacles - - timeout_s = 13.0 * 1.5 - files = [ - "--include=basics/(b|int_)", - "--exclude=basics/builtin_pow", - ] - - --> error after 20s - - ------------------ - - 5 tentacles - - timeout_s = 240.0 * 1.5 - files = ["--exclude=ports/rp2/rp2_lightsleep_thread.py"] # Broken test - - --> no error! - """ SUBPROCESS_INFRA_MPREMOTE = enum.auto() - """ - 13 tentacles - - timeout_s = 240.0 * 1.5 - files = ["--exclude=ports/rp2/rp2_lightsleep_thread.py"] # Broken test - - --> no error! - """ SUBPROCESS_INFRA_MPREMOTE_C = enum.auto() - """ - 13 tentacles - - timeout_s = 240.0 * 1.5 - files = ["--exclude=ports/rp2/rp2_lightsleep_thread.py"] # Broken test - - mpremote_c was called 3147 times! - --> no error! - """ class StressThread(threading.Thread): def __init__( self, scenario: EnumScenario, + stress_tentacle_count: int, tentacles_stress: ConnectedTentacles, directory_results: pathlib.Path, ): assert isinstance(scenario, EnumScenario) + assert isinstance(stress_tentacle_count, int) assert isinstance(tentacles_stress, ConnectedTentacles) assert isinstance(directory_results, pathlib.Path) super().__init__(daemon=True, name="stress") self._stopping = False + self._stress_tentacle_count = stress_tentacle_count self._scenario = scenario - self._tentacles_stress = tentacles_stress self._directory_results = directory_results + print( + f"Found {len(tentacles_stress)} tentacle to create stress. stress_tentacle_count={self._stress_tentacle_count}." + ) + self._tentacles_stress = tentacles_stress[: self._stress_tentacle_count] + print( + f"Tentacles to generate stress: {[serial_short_from_delimited(t.tentacle_instance.serial) for t in self._tentacles_stress]}" + ) def run(self) -> None: """ Power up all duts on all tentacles. Now loop over all tentacles and power down dut for a short time """ - print(f"Found {len(self._tentacles_stress)} tentacle to create stress.") - if self._scenario is EnumScenario.NONE: return self._scenario_NONE() @@ -179,16 +140,19 @@ def _scenario_INFRA_MPREMOTE(self) -> None: # continue serial_closed = t.infra.mp_remote_close() t.infra.connect_mpremote_if_needed() - print( - i, + fds = ( t.infra._mp_remote.state.transport.serial.fd, t.infra._mp_remote.state.transport.serial.pipe_abort_read_r, t.infra._mp_remote.state.transport.serial.pipe_abort_read_w, t.infra._mp_remote.state.transport.serial.pipe_abort_write_r, t.infra._mp_remote.state.transport.serial.pipe_abort_write_w, - end="", ) - print(" ", serial_closed, t.infra.mp_remote._tty) + print( + f"count={i:03d}", + f"pyserial.fds:{fds}", + f"close:{serial_closed}", + f"open:{t.infra.mp_remote._tty}", + ) rc = t.infra.mp_remote.exec_raw("print('Hello')") assert rc == "Hello\r\n" # print(rc) diff --git a/src/testbed_micropython/mpstress/util_test_run.py b/src/testbed_micropython/mpstress/util_test_run.py index 6a91e50..4c46029 100644 --- a/src/testbed_micropython/mpstress/util_test_run.py +++ b/src/testbed_micropython/mpstress/util_test_run.py @@ -26,31 +26,45 @@ class EnumTest(enum.StrEnum): RUN_TESTS_BASIC_B_INT = enum.auto() RUN_TESTS_BASIC_B_INT_POW = enum.auto() RUN_TESTS_EXTMOD = enum.auto() + SERIAL_TEST = enum.auto() @property def test_params(self) -> TestArgs: if self is EnumTest.RUN_TESTS_ALL: return TestArgs( timeout_s=240.0 * 1.5, - files=["--exclude=ports/rp2/rp2_lightsleep_thread.py"], # Broken test + program=["run-tests.py", "--jobs=1"], + files=[ + "--exclude=ports/rp2/rp2_lightsleep_thread.py", # Broken test + ], ) if self is EnumTest.RUN_TESTS_BASIC: return TestArgs( timeout_s=61.0 * 1.5, - files=["--include=basics/*"], + program=["run-tests.py", "--jobs=1"], + files=[ + "--include=basics/*", + ], ) if self is EnumTest.RUN_TESTS_EXTMOD: return TestArgs( timeout_s=77.0 * 1.5, - files=["--include=extmod/*"], + program=["run-tests.py", "--jobs=1"], + files=[ + "--include=extmod/*", + ], ) if self is EnumTest.RUN_TESTS_BASIC_B: return TestArgs( + program=["run-tests.py", "--jobs=1"], timeout_s=17.0 * 1.5, - files=["--include=basics/b"], + files=[ + "--include=basics/b", + ], ) if self is EnumTest.RUN_TESTS_BASIC_B_INT: return TestArgs( + program=["run-tests.py", "--jobs=1"], timeout_s=22.0 * 1.5, files=[ "--include=basics/(b|int_)", @@ -58,17 +72,25 @@ def test_params(self) -> TestArgs: ) if self is EnumTest.RUN_TESTS_BASIC_B_INT_POW: return TestArgs( + program=["run-tests.py", "--jobs=1"], timeout_s=13.0 * 1.5, files=[ "--include=basics/(b|int_)", "--exclude=basics/builtin_pow", ], ) + if self is EnumTest.SERIAL_TEST: + return TestArgs( + program=["serial_test.py", "--time-per-subtest=10"], + timeout_s=90.0 * 1.5, + files=[], + ) raise ValueError(self) @dataclasses.dataclass(repr=True, frozen=True) class TestArgs: + program: list[str] timeout_s: float files: list[str] @@ -92,12 +114,15 @@ def run_test( test_params = test.test_params + if test == EnumTest.SERIAL_TEST: + args_aux = [] + else: + args_aux = [f"--result-dir={directory_results}"] args = [ sys.executable, - "run-tests.py", - f"--result-dir={directory_results}", + *test_params.program, f"--test-instance=port:{tty}", - "--jobs=1", + *args_aux, *test_params.files, # "misc/cexample_class.py", ] @@ -110,5 +135,5 @@ def run_test( logfile=directory_results / "testresults.txt", timeout_s=test_params.timeout_s, # TODO: Remove the following line as soon returncode of 'run-multitest.py' is fixed. - success_returncodes=[0, 1], + # success_returncodes=[0, 1], ) From d0dd4d37eb6bede9ee4b996ddeb7ee7b35268c40 Mon Sep 17 00:00:00 2001 From: Hans Maerki Date: Wed, 22 Oct 2025 15:13:35 +0200 Subject: [PATCH 07/15] New: SIMPLE_SERIAL_WRITE --- .vscode/launch.json | 21 ++- .../mpstress/README_flakyness.md | 39 +++++ src/testbed_micropython/mpstress/cli.py | 2 +- .../mpstress/simple_serial_write.py | 145 ++++++++++++++++++ .../mpstress/util_stress.py | 18 ++- .../mpstress/util_test_run.py | 16 +- 6 files changed, 234 insertions(+), 7 deletions(-) create mode 100644 src/testbed_micropython/mpstress/simple_serial_write.py diff --git a/.vscode/launch.json b/.vscode/launch.json index d9cf19b..1b8b96e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -230,8 +230,9 @@ // "--scenario=SUBPROCESS_INFRA_MPREMOTE", // "--scenario=SUBPROCESS_INFRA_MPREMOTE_C", // "--test=RUN_TESTS_BASIC_B_INT_POW", - "--test=SERIAL_TEST", - "--stress-tentacle-count=10", + // "--test=SERIAL_TEST", + "--test=SIMPLE_SERIAL_WRITE", + "--stress-tentacle-count=99", // "--tentacle=1722", "--tentacle=5f2c", ], @@ -240,6 +241,22 @@ "PYDEVD_DISABLE_FILE_VALIDATION": "1", }, "justMyCode": false, + "subProcess": true, + }, + { + "name": "simple_serial_write", + "type": "debugpy", + "request": "launch", + "module": "testbed_micropython.mpstress.micropython.simple_serial_write", + "cwd": "${workspaceFolder}", + "args": [ + "--test-instance=port:/dev/ttyACM3" + ], + "console": "integratedTerminal", + "env": { + "PYDEVD_DISABLE_FILE_VALIDATION": "1", + }, + "justMyCode": false, "subProcess": false, }, { diff --git a/src/testbed_micropython/mpstress/README_flakyness.md b/src/testbed_micropython/mpstress/README_flakyness.md index d47fd26..63c893f 100644 --- a/src/testbed_micropython/mpstress/README_flakyness.md +++ b/src/testbed_micropython/mpstress/README_flakyness.md @@ -49,3 +49,42 @@ 6 tentacles --> no error! + +### --scenario=INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE + +12 tentacles +--> error after 4s + 006000: 197kBytes/s + ERROR, read_duration_s=1.001166s + expected: b'_ABCDEFGHIJKLMNOPQRSTUVWXYabcdefghijklmnopqrstuvwxy1234567890_' + received: b'_ABCDEFGHIJKLMNOPQRSTUVWXYabcdefghijklmnopqrstuvwxy123456789' + try reading again: b'' read_duration_s=1.001263s + + # Debug output from serialposix.py: + read(6(6)) duration=0.000011s + [4] = select(1.000s) duration=0.000018s + read(44(62)) duration=0.000010s + [] = select(1.000s) duration=1.000669s + TIMEOUT! + ERROR, read_duration_s=1.000970s + expected: b'_ABCDEFGHIJKLMNOPQRSTUVWXYabcdefghijklmnopqrstuvwxy1234567890_' + received: b'_ABCDEFGHIJKLMNOPQRSTUVWXYabcdefghijklmnopqr' + [] = select(1.000s) duration=1.000536s + TIMEOUT! + try reading again: b'' read_duration_s=1.008089s + + +### --scenario=INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE --stress-tentacle-count=8 + +8 tentacles +--> error after 9s, 19s + +### --scenario=INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE --stress-tentacle-count=7 + +7 tentacles +--> error after 5s, 25s + +### --scenario=NONE --test=SIMPLE_SERIAL_WRITE + +12 tentacles +--> no error diff --git a/src/testbed_micropython/mpstress/cli.py b/src/testbed_micropython/mpstress/cli.py index d8b19b2..d248c1d 100644 --- a/src/testbed_micropython/mpstress/cli.py +++ b/src/testbed_micropython/mpstress/cli.py @@ -114,7 +114,7 @@ def stress( for t in connected_tentacles: t.power.set_default_infra_on() - repo_micropython_tests = pathlib.Path(micropython_tests).resolve() + repo_micropython_tests = pathlib.Path(micropython_tests).expanduser().resolve() assert repo_micropython_tests.is_dir(), repo_micropython_tests st = StressThread( diff --git a/src/testbed_micropython/mpstress/simple_serial_write.py b/src/testbed_micropython/mpstress/simple_serial_write.py new file mode 100644 index 0000000..350210d --- /dev/null +++ b/src/testbed_micropython/mpstress/simple_serial_write.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +import dataclasses +import logging +import os +import time + +import serial +import typer +import typing_extensions + +from testbed_micropython.mpstress.util_stress import print_fds + +logger = logging.getLogger(__file__) + +# 'typer' does not work correctly with typing.Annotated +# Required is: typing_extensions.Annotated +TyperAnnotated = typing_extensions.Annotated + +# mypy: disable-error-code="valid-type" +# This will disable this warning: +# op.py:58: error: Variable "octoprobe.scripts.op.TyperAnnotated" is not valid as a type [valid-type] +# op.py:58: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases + +app = typer.Typer(pretty_exceptions_enable=False) + +CHARS = b"_ABCDEFGHIJKLMNOPQRSTUVWXYabcdefghijklmnopqrstuvwxy1234567890_" + +MICROPYTHON_SCRIPT = """ +# Based on +# /tests/serial_test.py: read_test_script + +import sys + +CHARS = b"_ABCDEFGHIJKLMNOPQRSTUVWXYabcdefghijklmnopqrstuvwxy1234567890_" + +def send_alphabet(count): + for i in range(count): + sys.stdout.buffer.write(CHARS) + sys.stdout.buffer.write(f"{i:06d}") + +send_alphabet(count=) +""" + + +class TestError(Exception): + pass + + +@dataclasses.dataclass(repr=True, frozen=True) +class SimpleSerialWrite: + serial: serial.Serial + + def drain_input(self): + time.sleep(0.1) + while self.serial.inWaiting() > 0: + _data = self.serial.read(self.serial.inWaiting()) + time.sleep(0.1) + + def send_script(self, script): + assert isinstance(script, bytes) + chunk_size = 32 + for i in range(0, len(script), chunk_size): + self.serial.write(script[i : i + chunk_size]) + time.sleep(0.01) + self.serial.write(b"\x04") # eof + self.serial.flush() + response = self.serial.read(2) + if response != b"OK": + response += self.serial.read(self.serial.inWaiting()) + raise TestError("could not send script", response) + + def read_test(self, count0: int): + self.serial.write(b"\x03\x01\x04") # break, raw-repl, soft-reboot + self.drain_input() + script = MICROPYTHON_SCRIPT.replace("", str(count0)) + self.send_script(script.encode("ascii")) + + start_s = time.monotonic() + byte_count = 0 + for i0 in range(count0): + i1 = i0 + 1 + read_start_s = time.monotonic() + chars = self.serial.read(len(CHARS)) + read_duration_s = time.monotonic() - read_start_s + if chars != CHARS: + print(f"ERROR, read_duration_s={read_duration_s:0.6f}s") + print(" expected:", CHARS) + print(" received:", chars) + read_start_s = time.monotonic() + chars2 = self.serial.read(len(CHARS)) + read_duration_s = time.monotonic() - read_start_s + print( + f" try reading again: {chars2} read_duration_s={read_duration_s:0.6f}s" + ) + print(f" serial:{self.serial!r}") + elements = [ + f"{self.serial.fd=}", + f"{self.serial.in_waiting=}", + f"{self.serial.pipe_abort_read_r=}", + f"{self.serial.pipe_abort_read_w=}", + f"{self.serial.pipe_abort_write_r=}", + f"{self.serial.pipe_abort_write_w=}", + f"{self.serial._rts_state=}", + f"{self.serial._break_state=}", + f"{self.serial._dtr_state=}", + f"{self.serial._dsrdtr=}", + ] + print(" ", " ".join(elements)) + print(f" {os.fstat(self.serial.fd)=!r}") + + print_fds() + + raise TestError("Received erronous data.") + count_text = self.serial.read(6) + count0 = int(count_text) + assert count0 == i0 + byte_count += len(CHARS) + 6 + if (i1 % 1000) == 0: + duration_s = time.monotonic() - start_s + # print(i, chars, count, f"{byte_count / duration_s / 1_000:0.6f}kBytes/s") + print(f"{i1:06d}: {byte_count / duration_s / 1_000:03.0f}kBytes/s") + start_s = time.monotonic() + byte_count = 0 + + +@app.command(help="Write datastream from micropython ") +def write_alphabet( + test_instance: TyperAnnotated[ + str, + typer.Option(help="For example port:/dev/ttyACM3"), + ] = None, # noqa: UP007 + count: TyperAnnotated[ + int, + typer.Option(help="The count of ~60byte-strings to be sent"), + ] = 10_000, # noqa: UP007 +) -> None: + assert test_instance.startswith("port:"), test_instance + serial_port = test_instance[len("port:") :] + ssw = SimpleSerialWrite(serial.Serial(serial_port, baudrate=115200, timeout=1)) + ssw.read_test(count0=count) + + +if __name__ == "__main__": + app() diff --git a/src/testbed_micropython/mpstress/util_stress.py b/src/testbed_micropython/mpstress/util_stress.py index b295c08..7f51eaf 100644 --- a/src/testbed_micropython/mpstress/util_stress.py +++ b/src/testbed_micropython/mpstress/util_stress.py @@ -2,7 +2,9 @@ import enum import logging +import os import pathlib +import subprocess import sys import threading import time @@ -19,6 +21,13 @@ logger = logging.getLogger(__file__) +def print_fds(): + cmd = f"ls -l /proc/{os.getpid()}/fd" + fd_text = subprocess.check_output(cmd, shell=True) + print(f" {cmd}: {len(fd_text.splitlines())} lines") + print(fd_text.decode("ascii")) + + class EnumScenario(enum.StrEnum): NONE = enum.auto() DUT_ON_OFF = enum.auto() @@ -72,7 +81,7 @@ def run(self) -> None: if self._scenario is EnumScenario.SUBPROCESS_INFRA_MPREMOTE_C: return self._scenario_SUBPROCESS_INFRA_MPREMOTE_C() - assert False + raise ValueError(f"Not handled: scenario {self._scenario}!") def _scenario_NONE(self) -> None: return @@ -132,6 +141,8 @@ def _scenario_INFRA_MPREMOTE(self) -> None: i = 0 while True: print("cycle") + print_fds() + for idx, t in enumerate(self._tentacles_stress): if self._stopping: return @@ -153,8 +164,9 @@ def _scenario_INFRA_MPREMOTE(self) -> None: f"close:{serial_closed}", f"open:{t.infra.mp_remote._tty}", ) - rc = t.infra.mp_remote.exec_raw("print('Hello')") - assert rc == "Hello\r\n" + if False: + rc = t.infra.mp_remote.exec_raw("print('Hello')") + assert rc == "Hello\r\n" # print(rc) def _scenario_DUT_ON_OFF(self) -> None: diff --git a/src/testbed_micropython/mpstress/util_test_run.py b/src/testbed_micropython/mpstress/util_test_run.py index 4c46029..84663a9 100644 --- a/src/testbed_micropython/mpstress/util_test_run.py +++ b/src/testbed_micropython/mpstress/util_test_run.py @@ -16,6 +16,8 @@ MICROPYTHON_DIRECTORY_TESTS, ) +DIRECTORY_OF_THIS_FILE = pathlib.Path(__file__).parent + logger = logging.getLogger(__file__) @@ -27,6 +29,7 @@ class EnumTest(enum.StrEnum): RUN_TESTS_BASIC_B_INT_POW = enum.auto() RUN_TESTS_EXTMOD = enum.auto() SERIAL_TEST = enum.auto() + SIMPLE_SERIAL_WRITE = enum.auto() @property def test_params(self) -> TestArgs: @@ -85,6 +88,12 @@ def test_params(self) -> TestArgs: timeout_s=90.0 * 1.5, files=[], ) + if self is EnumTest.SIMPLE_SERIAL_WRITE: + return TestArgs( + program=["simple_serial_write.py", "--count=1000000"], + timeout_s=340.0 * 1.5, + files=[], + ) raise ValueError(self) @@ -116,8 +125,13 @@ def run_test( if test == EnumTest.SERIAL_TEST: args_aux = [] + cwd = repo_micropython_tests / MICROPYTHON_DIRECTORY_TESTS + if test == EnumTest.SIMPLE_SERIAL_WRITE: + args_aux = [] + cwd = DIRECTORY_OF_THIS_FILE else: args_aux = [f"--result-dir={directory_results}"] + cwd = repo_micropython_tests / MICROPYTHON_DIRECTORY_TESTS args = [ sys.executable, *test_params.program, @@ -129,7 +143,7 @@ def run_test( env = ENV_PYTHONUNBUFFERED subprocess_run( args=args, - cwd=repo_micropython_tests / MICROPYTHON_DIRECTORY_TESTS, + cwd=cwd, env=env, # logfile=testresults_directory(f"run-tests-{test_dir}.txt").filename, logfile=directory_results / "testresults.txt", From 959c500c8d01b56550b080a30e9a78ce9976d430 Mon Sep 17 00:00:00 2001 From: Hans Maerki Date: Tue, 4 Nov 2025 09:03:25 +0100 Subject: [PATCH 08/15] Add README_flakyness.md --- .../mpstress/README_flakyness.md | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/testbed_micropython/mpstress/README_flakyness.md b/src/testbed_micropython/mpstress/README_flakyness.md index 63c893f..3c0e62f 100644 --- a/src/testbed_micropython/mpstress/README_flakyness.md +++ b/src/testbed_micropython/mpstress/README_flakyness.md @@ -52,6 +52,8 @@ ### --scenario=INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE +`mpstress --scenario=INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE --tentacle=5f2c --micropython-tests=/home/octoprobe/gits/micropython` + 12 tentacles --> error after 4s 006000: 197kBytes/s @@ -88,3 +90,21 @@ 12 tentacles --> no error + + + +### --scenario=INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE + +==> 5f2c connected to RHS B7 +`mpstress --scenario=INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE --tentacle=5f2c --micropython-tests=/home/octoprobe/gits/micropython` + +12 tentacles +--> error after 4s + +==> 5f2c connected to USB on computer rear + +`mpstress --scenario=INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE --tentacle=5f2c --micropython-tests=/home/octoprobe/gits/micropython` + +12 tentacles +--> error after 120s, 35s +--> no error 340s, 340s From 460e6238dbdefc4bfabb6905de95be20ec4e2ebb Mon Sep 17 00:00:00 2001 From: Hans Maerki Date: Sat, 27 Dec 2025 15:04:05 +0100 Subject: [PATCH 09/15] fixed all static check warnings --- src/testbed_micropython/mpstress/cli.py | 20 +++++++----- .../mpstress/simple_serial_write.py | 31 ++++++++++--------- .../mpstress/util_stress.py | 28 +++++++++++------ .../mpstress/util_test_run.py | 3 +- 4 files changed, 48 insertions(+), 34 deletions(-) diff --git a/src/testbed_micropython/mpstress/cli.py b/src/testbed_micropython/mpstress/cli.py index d248c1d..d6bdd3b 100644 --- a/src/testbed_micropython/mpstress/cli.py +++ b/src/testbed_micropython/mpstress/cli.py @@ -15,6 +15,7 @@ import typer import typing_extensions +from octoprobe import util_baseclasses from octoprobe.scripts.commissioning import init_logging from octoprobe.usb_tentacle.usb_tentacle import serial_short_from_delimited @@ -42,19 +43,21 @@ DIRECTORY_OF_THIS_FILE = pathlib.Path(__file__).parent DIRECTORY_RESULTS = DIRECTORY_OF_THIS_FILE / "testresults" DIRECTORY_RESULTS.mkdir(parents=True, exist_ok=True) -[f.unlink(missing_ok=True) for f in DIRECTORY_RESULTS.glob("*.txt")] -[f.unlink(missing_ok=True) for f in DIRECTORY_RESULTS.glob("*.out")] +for f in DIRECTORY_RESULTS.glob("*.txt"): + f.unlink(missing_ok=True) +for f in DIRECTORY_RESULTS.glob("*.out"): + f.unlink(missing_ok=True) -def complete_scenario(): +def complete_scenario() -> list[str]: return sorted([scenario.name for scenario in EnumScenario]) -def complete_test(): +def complete_test() -> list[str]: return sorted([test.name for test in EnumTest]) -def complete_only_tentacle(): +def complete_only_tentacle() -> list[str]: connected_tentacles = util_testrunner.query_connected_tentacles_fast() return sorted( @@ -106,13 +109,14 @@ def stress( tentacles_load: ConnectedTentacles = ConnectedTentacles() for t in connected_tentacles: serial_short = serial_short_from_delimited(t.tentacle_instance.serial) - if serial_short in tentacle: + if serial_short in tentacle: # type: ignore[attr-defined] tentacle_test = t continue tentacles_load.append(t) assert tentacle_test is not None, f"Tentacle not connected: {tentacle}" for t in connected_tentacles: - t.power.set_default_infra_on() + t.infra.load_base_code_if_needed() + t.switches.default_off_infra_on() repo_micropython_tests = pathlib.Path(micropython_tests).expanduser().resolve() assert repo_micropython_tests.is_dir(), repo_micropython_tests @@ -132,7 +136,7 @@ def stress( ) st.stop() - except util_testrunner.OctoprobeAppExitException as e: + except util_baseclasses.OctoprobeAppExitException as e: logger.info(f"Terminating test due to OctoprobeAppExitException: {e}") raise typer.Exit(1) from e diff --git a/src/testbed_micropython/mpstress/simple_serial_write.py b/src/testbed_micropython/mpstress/simple_serial_write.py index 350210d..9e552be 100644 --- a/src/testbed_micropython/mpstress/simple_serial_write.py +++ b/src/testbed_micropython/mpstress/simple_serial_write.py @@ -51,13 +51,13 @@ class TestError(Exception): class SimpleSerialWrite: serial: serial.Serial - def drain_input(self): + def drain_input(self) -> None: time.sleep(0.1) - while self.serial.inWaiting() > 0: - _data = self.serial.read(self.serial.inWaiting()) + while self.serial.inWaiting() > 0: # type: ignore[attr-defined] + _data = self.serial.read(self.serial.inWaiting()) # type: ignore[attr-defined] time.sleep(0.1) - def send_script(self, script): + def send_script(self, script: bytes) -> None: assert isinstance(script, bytes) chunk_size = 32 for i in range(0, len(script), chunk_size): @@ -67,10 +67,10 @@ def send_script(self, script): self.serial.flush() response = self.serial.read(2) if response != b"OK": - response += self.serial.read(self.serial.inWaiting()) + response += self.serial.read(self.serial.inWaiting()) # type: ignore[attr-defined] raise TestError("could not send script", response) - def read_test(self, count0: int): + def read_test(self, count0: int) -> None: self.serial.write(b"\x03\x01\x04") # break, raw-repl, soft-reboot self.drain_input() script = MICROPYTHON_SCRIPT.replace("", str(count0)) @@ -91,7 +91,7 @@ def read_test(self, count0: int): chars2 = self.serial.read(len(CHARS)) read_duration_s = time.monotonic() - read_start_s print( - f" try reading again: {chars2} read_duration_s={read_duration_s:0.6f}s" + f" try reading again: {chars2.decode('ascii')} read_duration_s={read_duration_s:0.6f}s" ) print(f" serial:{self.serial!r}") elements = [ @@ -101,12 +101,13 @@ def read_test(self, count0: int): f"{self.serial.pipe_abort_read_w=}", f"{self.serial.pipe_abort_write_r=}", f"{self.serial.pipe_abort_write_w=}", - f"{self.serial._rts_state=}", - f"{self.serial._break_state=}", - f"{self.serial._dtr_state=}", - f"{self.serial._dsrdtr=}", + f"{self.serial._rts_state=}", # type: ignore[attr-defined] + f"{self.serial._break_state=}", # type: ignore[attr-defined] + f"{self.serial._dtr_state=}", # type: ignore[attr-defined] + f"{self.serial._dsrdtr=}", # type: ignore[attr-defined] ] print(" ", " ".join(elements)) + assert isinstance(self.serial.fd, int), self.serial.fd print(f" {os.fstat(self.serial.fd)=!r}") print_fds() @@ -129,14 +130,16 @@ def write_alphabet( test_instance: TyperAnnotated[ str, typer.Option(help="For example port:/dev/ttyACM3"), - ] = None, # noqa: UP007 + ], count: TyperAnnotated[ int, typer.Option(help="The count of ~60byte-strings to be sent"), ] = 10_000, # noqa: UP007 ) -> None: - assert test_instance.startswith("port:"), test_instance - serial_port = test_instance[len("port:") :] + assert isinstance(test_instance, str) + assert str(test_instance).startswith("port:"), test_instance + + serial_port = test_instance[len("port:") :] # type: ignore[index] ssw = SimpleSerialWrite(serial.Serial(serial_port, baudrate=115200, timeout=1)) ssw.read_test(count0=count) diff --git a/src/testbed_micropython/mpstress/util_stress.py b/src/testbed_micropython/mpstress/util_stress.py index 7f51eaf..9a4915e 100644 --- a/src/testbed_micropython/mpstress/util_stress.py +++ b/src/testbed_micropython/mpstress/util_stress.py @@ -20,8 +20,11 @@ logger = logging.getLogger(__file__) +# pylint: disable=invalid-name +# pylint: disable=no-member +# pylint: disable=protected-access -def print_fds(): +def print_fds() -> None: cmd = f"ls -l /proc/{os.getpid()}/fd" fd_text = subprocess.check_output(cmd, shell=True) print(f" {cmd}: {len(fd_text.splitlines())} lines") @@ -94,6 +97,7 @@ def _scenario_SUBPROCESS_INFRA_MPREMOTE_C(self) -> None: if self._stopping: return i += 1 + assert t.infra.usb_tentacle.serial_port is not None args = [ str(DIRECTORY_OF_THIS_FILE / "c" / "mpremote_c"), t.infra.usb_tentacle.serial_port, @@ -115,6 +119,7 @@ def _scenario_SUBPROCESS_INFRA_MPREMOTE(self) -> None: if self._stopping: return i += 1 + assert t.infra.usb_tentacle.serial_port is not None args = [ sys.executable, "-m", @@ -136,14 +141,14 @@ def _scenario_SUBPROCESS_INFRA_MPREMOTE(self) -> None: def _scenario_INFRA_MPREMOTE(self) -> None: print("off") for t in self._tentacles_stress: - t.infra.power.dut = False + t.infra.switches.dut = False i = 0 while True: print("cycle") print_fds() - for idx, t in enumerate(self._tentacles_stress): + for _idx, t in enumerate(self._tentacles_stress): if self._stopping: return i += 1 @@ -151,12 +156,15 @@ def _scenario_INFRA_MPREMOTE(self) -> None: # continue serial_closed = t.infra.mp_remote_close() t.infra.connect_mpremote_if_needed() + assert t.infra._mp_remote is not None + assert t.infra._mp_remote.state.transport is not None + serial = t.infra._mp_remote.state.transport.serial fds = ( - t.infra._mp_remote.state.transport.serial.fd, - t.infra._mp_remote.state.transport.serial.pipe_abort_read_r, - t.infra._mp_remote.state.transport.serial.pipe_abort_read_w, - t.infra._mp_remote.state.transport.serial.pipe_abort_write_r, - t.infra._mp_remote.state.transport.serial.pipe_abort_write_w, + serial.fd, + serial.pipe_abort_read_r, + serial.pipe_abort_read_w, + serial.pipe_abort_write_r, + serial.pipe_abort_write_w, ) print( f"count={i:03d}", @@ -174,12 +182,12 @@ def _scenario_DUT_ON_OFF(self) -> None: while not self._stopping: print("on") for t in self._tentacles_stress: - t.infra.power.dut = True + t.infra.switches.dut = True time.sleep(sleep_s) print("off") for t in self._tentacles_stress: - t.infra.power.dut = False + t.infra.switches.dut = False time.sleep(sleep_s) def stop(self) -> None: diff --git a/src/testbed_micropython/mpstress/util_test_run.py b/src/testbed_micropython/mpstress/util_test_run.py index 84663a9..0a11c7f 100644 --- a/src/testbed_micropython/mpstress/util_test_run.py +++ b/src/testbed_micropython/mpstress/util_test_run.py @@ -123,11 +123,10 @@ def run_test( test_params = test.test_params + args_aux: list[str] = [] if test == EnumTest.SERIAL_TEST: - args_aux = [] cwd = repo_micropython_tests / MICROPYTHON_DIRECTORY_TESTS if test == EnumTest.SIMPLE_SERIAL_WRITE: - args_aux = [] cwd = DIRECTORY_OF_THIS_FILE else: args_aux = [f"--result-dir={directory_results}"] From 87f7cff0cf00dc3f0576414e38c62c13de39a311 Mon Sep 17 00:00:00 2001 From: Hans Maerki Date: Sat, 27 Dec 2025 18:20:44 +0100 Subject: [PATCH 10/15] Reactivate --test=SERIAL_TEST --- src/testbed_micropython/mpstress/util_test_run.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/testbed_micropython/mpstress/util_test_run.py b/src/testbed_micropython/mpstress/util_test_run.py index 0a11c7f..c641470 100644 --- a/src/testbed_micropython/mpstress/util_test_run.py +++ b/src/testbed_micropython/mpstress/util_test_run.py @@ -124,13 +124,13 @@ def run_test( test_params = test.test_params args_aux: list[str] = [] + cwd = repo_micropython_tests / MICROPYTHON_DIRECTORY_TESTS if test == EnumTest.SERIAL_TEST: - cwd = repo_micropython_tests / MICROPYTHON_DIRECTORY_TESTS - if test == EnumTest.SIMPLE_SERIAL_WRITE: + pass + elif test == EnumTest.SIMPLE_SERIAL_WRITE: cwd = DIRECTORY_OF_THIS_FILE else: args_aux = [f"--result-dir={directory_results}"] - cwd = repo_micropython_tests / MICROPYTHON_DIRECTORY_TESTS args = [ sys.executable, *test_params.program, @@ -140,6 +140,7 @@ def run_test( # "misc/cexample_class.py", ] env = ENV_PYTHONUNBUFFERED + print(f"RUN: run_test(): subprocess_run({args})") subprocess_run( args=args, cwd=cwd, @@ -150,3 +151,4 @@ def run_test( # TODO: Remove the following line as soon returncode of 'run-multitest.py' is fixed. # success_returncodes=[0, 1], ) + print(f"DONE: run_test(): subprocess_run({args})") From df6395f61cd0a0ca86242980e676b71a28521b55 Mon Sep 17 00:00:00 2001 From: Hans Maerki Date: Thu, 1 Jan 2026 13:16:03 +0100 Subject: [PATCH 11/15] Import mpstress --- .vscode/launch.json | 14 ++- src/testbed_micropython/mpstress/cli.py | 111 +++++++++++++----- .../mpstress/simple_serial_write.py | 5 +- .../mpstress/util_stress.py | 36 ++++-- .../mpstress/util_test_run.py | 19 ++- 5 files changed, 129 insertions(+), 56 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 1b8b96e..2a3387f 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -224,9 +224,9 @@ // "stress", // "--micropython-tests=${workspaceFolder}/../fork_micropython", "--micropython-tests=${workspaceFolder}/../micropython", - // "--scenario=NONE", + "--scenario=NONE", // "--scenario=DUT_ON_OFF", - "--scenario=INFRA_MPREMOTE", + // "--scenario=INFRA_MPREMOTE", // "--scenario=SUBPROCESS_INFRA_MPREMOTE", // "--scenario=SUBPROCESS_INFRA_MPREMOTE_C", // "--test=RUN_TESTS_BASIC_B_INT_POW", @@ -234,7 +234,10 @@ "--test=SIMPLE_SERIAL_WRITE", "--stress-tentacle-count=99", // "--tentacle=1722", - "--tentacle=5f2c", + // "--tentacle=5f2a", // 5f2a-ADA_ITSYBITSY_M0 + // "--tentacle=5f2c", // 5f2c-RPI_PICO_W + // "--tentacle=3c2a", // 3c2a-ARDUINO_NANO_33 + "--tentacle=0c30", // 0c30-ESP32_C3_DEVKIT ], "console": "integratedTerminal", "env": { @@ -247,10 +250,11 @@ "name": "simple_serial_write", "type": "debugpy", "request": "launch", - "module": "testbed_micropython.mpstress.micropython.simple_serial_write", + "module": "testbed_micropython.mpstress.simple_serial_write", "cwd": "${workspaceFolder}", "args": [ - "--test-instance=port:/dev/ttyACM3" + "--count=10000", + "--test-instance=port:/dev/ttyUSB0" ], "console": "integratedTerminal", "env": { diff --git a/src/testbed_micropython/mpstress/cli.py b/src/testbed_micropython/mpstress/cli.py index d6bdd3b..46f88ca 100644 --- a/src/testbed_micropython/mpstress/cli.py +++ b/src/testbed_micropython/mpstress/cli.py @@ -78,7 +78,7 @@ def stress( ), ] = constants.URL_FILENAME_DEFAULT, tentacle: TyperAnnotated[ - list[str], + str | None, typer.Option( help="Run tests only on these tentacles. All other tentacles are used to create stress", autocompletion=complete_only_tentacle, @@ -102,44 +102,91 @@ def stress( ] = EnumTest.RUN_TESTS_BASIC_B_INT_POW, ) -> None: init_logging() + connected_tentacles = util_testrunner.query_connected_tentacles_fast() + connected_tentacles.sort( + key=lambda t: (t.tentacle_spec_base.tentacle_tag, t.tentacle_serial_number) + ) try: - connected_tentacles = util_testrunner.query_connected_tentacles_fast() - tentacle_test: TentacleMicropython | None = None - tentacles_load: ConnectedTentacles = ConnectedTentacles() - for t in connected_tentacles: - serial_short = serial_short_from_delimited(t.tentacle_instance.serial) - if serial_short in tentacle: # type: ignore[attr-defined] - tentacle_test = t - continue - tentacles_load.append(t) - assert tentacle_test is not None, f"Tentacle not connected: {tentacle}" - for t in connected_tentacles: - t.infra.load_base_code_if_needed() - t.switches.default_off_infra_on() - - repo_micropython_tests = pathlib.Path(micropython_tests).expanduser().resolve() - assert repo_micropython_tests.is_dir(), repo_micropython_tests - - st = StressThread( - scenario=EnumScenario[scenario], - stress_tentacle_count=stress_tentacle_count, - tentacles_stress=tentacles_load, - directory_results=DIRECTORY_RESULTS, - ) - st.start() - run_test( - tentacle_test=tentacle_test, - repo_micropython_tests=repo_micropython_tests, - directory_results=DIRECTORY_RESULTS, - test=EnumTest[test], - ) - st.stop() + if tentacle is not None: + test_one_tentacle( + connected_tentacles=connected_tentacles, + micropython_tests=micropython_tests, + tentacle=tentacle, + stress_tentacle_count=stress_tentacle_count, + scenario=scenario, + test=test, + ) + else: + for connected_tentacle in connected_tentacles: + test_one_tentacle( + connected_tentacles=connected_tentacles, + micropython_tests=micropython_tests, + tentacle=serial_short_from_delimited( + connected_tentacle.tentacle_serial_number + ), + stress_tentacle_count=stress_tentacle_count, + scenario=scenario, + test=test, + ) except util_baseclasses.OctoprobeAppExitException as e: logger.info(f"Terminating test due to OctoprobeAppExitException: {e}") raise typer.Exit(1) from e +def test_one_tentacle( + connected_tentacles: ConnectedTentacles, + micropython_tests: str, + tentacle: str, + stress_tentacle_count: int, + scenario: str, + test: str, +): + scenario = EnumScenario[scenario] + tentacle_test: TentacleMicropython | None = None + tentacles_load: ConnectedTentacles = ConnectedTentacles() + for t in connected_tentacles: + serial_short = serial_short_from_delimited(t.tentacle_serial_number) + if serial_short == tentacle: # type: ignore[attr-defined] + tentacle_test = t + continue + tentacles_load.append(t) + + assert tentacle_test is not None, f"Tentacle not connected: {tentacle}" + print(f"*** Initialized {len(connected_tentacles)} tentacles") + for t in connected_tentacles: + t.infra.load_base_code_if_needed() + t.switches.default_off_infra_on() + if scenario is EnumScenario.INFRA_MPREMOTE: + # if t == tentacle_test: + # t.infra.switches.dut = True + # else: + # t.infra.switches.dut = False + t.infra.switches.dut = False + + repo_micropython_tests = pathlib.Path(micropython_tests).expanduser().resolve() + assert repo_micropython_tests.is_dir(), repo_micropython_tests + + st = StressThread( + scenario=scenario, + stress_tentacle_count=stress_tentacle_count, + tentacles_stress=tentacles_load, + directory_results=DIRECTORY_RESULTS, + ) + + print("*** start") + st.start() + print("*** run_test") + run_test( + tentacle_test=tentacle_test, + repo_micropython_tests=repo_micropython_tests, + directory_results=DIRECTORY_RESULTS, + test=EnumTest[test], + ) + print("*** stop") + st.stop() + + if __name__ == "__main__": app() diff --git a/src/testbed_micropython/mpstress/simple_serial_write.py b/src/testbed_micropython/mpstress/simple_serial_write.py index 9e552be..4eb9f14 100644 --- a/src/testbed_micropython/mpstress/simple_serial_write.py +++ b/src/testbed_micropython/mpstress/simple_serial_write.py @@ -37,7 +37,8 @@ def send_alphabet(count): for i in range(count): sys.stdout.buffer.write(CHARS) - sys.stdout.buffer.write(f"{i:06d}") + # sys.stdout.buffer.write(f"{i:06d}") + sys.stdout.buffer.write("%06d" % i) send_alphabet(count=) """ @@ -140,7 +141,7 @@ def write_alphabet( assert str(test_instance).startswith("port:"), test_instance serial_port = test_instance[len("port:") :] # type: ignore[index] - ssw = SimpleSerialWrite(serial.Serial(serial_port, baudrate=115200, timeout=1)) + ssw = SimpleSerialWrite(serial.Serial(serial_port, baudrate=115200, timeout=5)) ssw.read_test(count0=count) diff --git a/src/testbed_micropython/mpstress/util_stress.py b/src/testbed_micropython/mpstress/util_stress.py index 9a4915e..6c6d391 100644 --- a/src/testbed_micropython/mpstress/util_stress.py +++ b/src/testbed_micropython/mpstress/util_stress.py @@ -20,10 +20,14 @@ logger = logging.getLogger(__file__) +PRINT_STRESS_OUTPUT = False +LOG_OUTPUT_S = 10.0 + # pylint: disable=invalid-name # pylint: disable=no-member # pylint: disable=protected-access + def print_fds() -> None: cmd = f"ls -l /proc/{os.getpid()}/fd" fd_text = subprocess.check_output(cmd, shell=True) @@ -52,6 +56,7 @@ def __init__( assert isinstance(tentacles_stress, ConnectedTentacles) assert isinstance(directory_results, pathlib.Path) super().__init__(daemon=True, name="stress") + self._next_log_output_s = time.monotonic() + LOG_OUTPUT_S self._stopping = False self._stress_tentacle_count = stress_tentacle_count self._scenario = scenario @@ -64,6 +69,13 @@ def __init__( f"Tentacles to generate stress: {[serial_short_from_delimited(t.tentacle_instance.serial) for t in self._tentacles_stress]}" ) + @property + def do_log_output(self) -> bool: + if time.monotonic() > self._next_log_output_s: + self._next_log_output_s += LOG_OUTPUT_S + return True + return False + def run(self) -> None: """ Power up all duts on all tentacles. @@ -139,14 +151,11 @@ def _scenario_SUBPROCESS_INFRA_MPREMOTE(self) -> None: ) def _scenario_INFRA_MPREMOTE(self) -> None: - print("off") - for t in self._tentacles_stress: - t.infra.switches.dut = False - i = 0 while True: - print("cycle") - print_fds() + if PRINT_STRESS_OUTPUT: + print("cycle") + print_fds() for _idx, t in enumerate(self._tentacles_stress): if self._stopping: @@ -166,15 +175,18 @@ def _scenario_INFRA_MPREMOTE(self) -> None: serial.pipe_abort_write_r, serial.pipe_abort_write_w, ) - print( - f"count={i:03d}", - f"pyserial.fds:{fds}", - f"close:{serial_closed}", - f"open:{t.infra.mp_remote._tty}", - ) + # if PRINT_STRESS_OUTPUT: + if self.do_log_output: + print( + f"count={i:03d}", + f"pyserial.fds:{fds}", + f"close:{serial_closed}", + f"open:{t.infra.mp_remote._tty}", + ) if False: rc = t.infra.mp_remote.exec_raw("print('Hello')") assert rc == "Hello\r\n" + time.sleep(0.001) # print(rc) def _scenario_DUT_ON_OFF(self) -> None: diff --git a/src/testbed_micropython/mpstress/util_test_run.py b/src/testbed_micropython/mpstress/util_test_run.py index c641470..b7534d3 100644 --- a/src/testbed_micropython/mpstress/util_test_run.py +++ b/src/testbed_micropython/mpstress/util_test_run.py @@ -89,9 +89,14 @@ def test_params(self) -> TestArgs: files=[], ) if self is EnumTest.SIMPLE_SERIAL_WRITE: + duration_factor = 1 + # duration_factor = 100 return TestArgs( - program=["simple_serial_write.py", "--count=1000000"], - timeout_s=340.0 * 1.5, + program=[ + "simple_serial_write.py", + f"--count={int(duration_factor * 10000)}", + ], + timeout_s=duration_factor * 3.4 * 1.5 + 10.0, files=[], ) raise ValueError(self) @@ -115,10 +120,13 @@ def run_test( assert isinstance(directory_results, pathlib.Path) assert isinstance(test, EnumTest) + print(f"*** power up tentacle_test: {tentacle_test.label_short}") + # tentacle_test.power.dut = True with UdevPoller() as udev: tty = tentacle_test.dut.dut_mcu.application_mode_power_up( - tentacle=tentacle_test, udev=udev + tentacle=tentacle_test, + udev=udev, ) test_params = test.test_params @@ -140,13 +148,14 @@ def run_test( # "misc/cexample_class.py", ] env = ENV_PYTHONUNBUFFERED - print(f"RUN: run_test(): subprocess_run({args})") + print(f"** RUN: run_test(): subprocess_run({args})") subprocess_run( args=args, cwd=cwd, env=env, # logfile=testresults_directory(f"run-tests-{test_dir}.txt").filename, - logfile=directory_results / "testresults.txt", + logfile=directory_results + / f"testresults_{tentacle_test.tentacle_serial_number}_{tentacle_test.tentacle_spec_base.tentacle_tag}.txt", timeout_s=test_params.timeout_s, # TODO: Remove the following line as soon returncode of 'run-multitest.py' is fixed. # success_returncodes=[0, 1], From 02a2b8547e32e87a8328926b59af7c2b3b03ec66 Mon Sep 17 00:00:00 2001 From: Hans Maerki Date: Fri, 2 Jan 2026 11:17:13 +0100 Subject: [PATCH 12/15] Stress succeeds --- .vscode/launch.json | 10 ++-- src/testbed_micropython/mpstress/cli.py | 20 ++++---- .../mpstress/simple_serial_write.py | 12 ++++- .../mpstress/util_test_run.py | 47 +++++++++++++++---- 4 files changed, 66 insertions(+), 23 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 2a3387f..3fcb883 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -146,13 +146,13 @@ // "--skip-test=RUN-MULTITESTS_MULTINET", // "--only-test=RUN-NATMODTESTS", // "--only-board=LOLIN_C3_MINI", - // "--only-board=LOLIN_D1_MINI", + "--only-board=LOLIN_D1_MINI", // "--only-fut=FUT_WLAN", // "--skip-fut=FUT_WLAN", // "--skip-fut=FUT_BLE", "--no-multiprocessing", // "--flash-force", - "--flash-skip", + // "--flash-skip", // "--count=24", // "--no-git-clean", // "--debug-skip-tests", @@ -233,11 +233,13 @@ // "--test=SERIAL_TEST", "--test=SIMPLE_SERIAL_WRITE", "--stress-tentacle-count=99", - // "--tentacle=1722", // "--tentacle=5f2a", // 5f2a-ADA_ITSYBITSY_M0 // "--tentacle=5f2c", // 5f2c-RPI_PICO_W // "--tentacle=3c2a", // 3c2a-ARDUINO_NANO_33 - "--tentacle=0c30", // 0c30-ESP32_C3_DEVKIT + // "--tentacle=0c30", // 0c30-ESP32_C3_DEVKIT + // "--tentacle=5d21", // 5d21-ESP32_DEVKIT + // "--tentacle=2d2d", // 2d2d_LOLIN_D1_MINI + // "--tentacle=3a21",// 3a21-PYBV11 ], "console": "integratedTerminal", "env": { diff --git a/src/testbed_micropython/mpstress/cli.py b/src/testbed_micropython/mpstress/cli.py index 46f88ca..2ce6fb9 100644 --- a/src/testbed_micropython/mpstress/cli.py +++ b/src/testbed_micropython/mpstress/cli.py @@ -103,9 +103,10 @@ def stress( ) -> None: init_logging() connected_tentacles = util_testrunner.query_connected_tentacles_fast() - connected_tentacles.sort( - key=lambda t: (t.tentacle_spec_base.tentacle_tag, t.tentacle_serial_number) - ) + # connected_tentacles.sort( + # key=lambda t: (t.tentacle_spec_base.tentacle_tag, t.tentacle_serial_number) + # ) + connected_tentacles.sort(key=lambda t: t.tentacle_serial_number) try: if tentacle is not None: @@ -156,14 +157,15 @@ def test_one_tentacle( assert tentacle_test is not None, f"Tentacle not connected: {tentacle}" print(f"*** Initialized {len(connected_tentacles)} tentacles") for t in connected_tentacles: + print(f"**** load_base_code_if_needed {t.description_short}") t.infra.load_base_code_if_needed() t.switches.default_off_infra_on() - if scenario is EnumScenario.INFRA_MPREMOTE: - # if t == tentacle_test: - # t.infra.switches.dut = True - # else: - # t.infra.switches.dut = False - t.infra.switches.dut = False + # if scenario is EnumScenario.INFRA_MPREMOTE: + # # if t == tentacle_test: + # # t.infra.switches.dut = True + # # else: + # # t.infra.switches.dut = False + # t.infra.switches.dut = False repo_micropython_tests = pathlib.Path(micropython_tests).expanduser().resolve() assert repo_micropython_tests.is_dir(), repo_micropython_tests diff --git a/src/testbed_micropython/mpstress/simple_serial_write.py b/src/testbed_micropython/mpstress/simple_serial_write.py index 4eb9f14..6c90e4a 100644 --- a/src/testbed_micropython/mpstress/simple_serial_write.py +++ b/src/testbed_micropython/mpstress/simple_serial_write.py @@ -34,11 +34,19 @@ CHARS = b"_ABCDEFGHIJKLMNOPQRSTUVWXYabcdefghijklmnopqrstuvwxy1234567890_" +def write(msg): + try: + sys.stdout.buffer.write(msg) + except AttributeError: + # 2d2d_LOLIN_D1_MINI + print(msg.decode("ascii"), end="") + def send_alphabet(count): for i in range(count): - sys.stdout.buffer.write(CHARS) + write(CHARS) + # 2d2d_LOLIN_D1_MINI # sys.stdout.buffer.write(f"{i:06d}") - sys.stdout.buffer.write("%06d" % i) + write(b"%06d" % i) send_alphabet(count=) """ diff --git a/src/testbed_micropython/mpstress/util_test_run.py b/src/testbed_micropython/mpstress/util_test_run.py index b7534d3..2314062 100644 --- a/src/testbed_micropython/mpstress/util_test_run.py +++ b/src/testbed_micropython/mpstress/util_test_run.py @@ -5,6 +5,7 @@ import logging import pathlib import sys +import time from octoprobe.util_pyudev import UdevPoller from octoprobe.util_subprocess import subprocess_run @@ -31,8 +32,22 @@ class EnumTest(enum.StrEnum): SERIAL_TEST = enum.auto() SIMPLE_SERIAL_WRITE = enum.auto() - @property - def test_params(self) -> TestArgs: + def get_test_args(self, tentacle_test: TentacleMicropython) -> TestArgs: + serial_speed_default = 249 # kByts/s + serial_speed = serial_speed_default + for mcu, _serial_speed in ( + ("ESP32", 12), + ("LOLIN_D1", 12), + ("WB55", 12), + ("ADA_ITSYBITSY_M0", 140), + ): + if mcu in tentacle_test.description_short: + serial_speed = _serial_speed + print( + f"*** {mcu}: serial_speed_default={serial_speed_default}, serial_speed={serial_speed}kBytes/s" + ) + break + if self is EnumTest.RUN_TESTS_ALL: return TestArgs( timeout_s=240.0 * 1.5, @@ -90,11 +105,15 @@ def test_params(self) -> TestArgs: ) if self is EnumTest.SIMPLE_SERIAL_WRITE: duration_factor = 1 + duration_factor = 5 + duration_factor = 2 # duration_factor = 100 + count = int(duration_factor * 10000 * serial_speed / serial_speed_default) + count = max(1000, count) return TestArgs( program=[ "simple_serial_write.py", - f"--count={int(duration_factor * 10000)}", + f"--count={count}", ], timeout_s=duration_factor * 3.4 * 1.5 + 10.0, files=[], @@ -128,8 +147,14 @@ def run_test( tentacle=tentacle_test, udev=udev, ) + time.sleep(1.0) - test_params = test.test_params + test_args = test.get_test_args(tentacle_test) + # timeout_s = test_params.timeout_s + # for mcu in ("ESP32", "LOLIN_D1", "WB55"): + # if mcu in tentacle_test.description_short: + # timeout_s *= 15 + # logger.info(f"*** {mcu}: timeout_s={timeout_s:0.0f}s") args_aux: list[str] = [] cwd = repo_micropython_tests / MICROPYTHON_DIRECTORY_TESTS @@ -141,14 +166,15 @@ def run_test( args_aux = [f"--result-dir={directory_results}"] args = [ sys.executable, - *test_params.program, + *test_args.program, f"--test-instance=port:{tty}", *args_aux, - *test_params.files, + *test_args.files, # "misc/cexample_class.py", ] env = ENV_PYTHONUNBUFFERED - print(f"** RUN: run_test(): subprocess_run({args})") + print(f"*** RUN: run_test(): subprocess_run({args})") + begin_s = time.monotonic() subprocess_run( args=args, cwd=cwd, @@ -156,8 +182,13 @@ def run_test( # logfile=testresults_directory(f"run-tests-{test_dir}.txt").filename, logfile=directory_results / f"testresults_{tentacle_test.tentacle_serial_number}_{tentacle_test.tentacle_spec_base.tentacle_tag}.txt", - timeout_s=test_params.timeout_s, + timeout_s=test_args.timeout_s, # TODO: Remove the following line as soon returncode of 'run-multitest.py' is fixed. # success_returncodes=[0, 1], ) + duration_s = time.monotonic() - begin_s + if duration_s > test_args.timeout_s * 0.5: + print( + f"*** WARNING: {tentacle_test.description_short}: duration_s={duration_s:0.0f}s > timeout*0.5={test_args.timeout_s * 0.5:0.0f}s" + ) print(f"DONE: run_test(): subprocess_run({args})") From c208d3269a73921fcfebb6225fa0cc54ae6a8066 Mon Sep 17 00:00:00 2001 From: Hans Maerki Date: Sun, 4 Jan 2026 08:55:20 +0100 Subject: [PATCH 13/15] scenario -> stress_scenario --- .gitignore | 2 +- .vscode/launch.json | 12 +++---- .../mpstress/README_flakyness.md | 36 +++++++++---------- src/testbed_micropython/mpstress/cli.py | 18 +++++----- .../mpstress/util_stress.py | 18 +++++----- .../mpstress/util_test_run.py | 5 --- 6 files changed, 44 insertions(+), 47 deletions(-) diff --git a/.gitignore b/.gitignore index 8451565..f10127a 100644 --- a/.gitignore +++ b/.gitignore @@ -14,5 +14,5 @@ src/testbed_micropython/mpbuild/tests/results/*.txt src/testbed_micropython/mpbuild/docker-build-micropython-esp32/esp-idf src/testbed/experiments/* testresults*/** -src/testbed_micropython/mpstress/testresults/ +src/testbed_micropython/mpstress/testresults*/** src/testbed_micropython/mpstress/c/mpremote_c diff --git a/.vscode/launch.json b/.vscode/launch.json index 3fcb883..12ed184 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -224,11 +224,11 @@ // "stress", // "--micropython-tests=${workspaceFolder}/../fork_micropython", "--micropython-tests=${workspaceFolder}/../micropython", - "--scenario=NONE", - // "--scenario=DUT_ON_OFF", - // "--scenario=INFRA_MPREMOTE", - // "--scenario=SUBPROCESS_INFRA_MPREMOTE", - // "--scenario=SUBPROCESS_INFRA_MPREMOTE_C", + // "--stress-scenario=NONE", + // "--stress-scenario=DUT_ON_OFF", + "--stress-scenario=INFRA_MPREMOTE", + // "--stress-scenario=SUBPROCESS_INFRA_MPREMOTE", + // "--stress-scenario=SUBPROCESS_INFRA_MPREMOTE_C", // "--test=RUN_TESTS_BASIC_B_INT_POW", // "--test=SERIAL_TEST", "--test=SIMPLE_SERIAL_WRITE", @@ -239,7 +239,7 @@ // "--tentacle=0c30", // 0c30-ESP32_C3_DEVKIT // "--tentacle=5d21", // 5d21-ESP32_DEVKIT // "--tentacle=2d2d", // 2d2d_LOLIN_D1_MINI - // "--tentacle=3a21",// 3a21-PYBV11 + // "--tentacle=3a21", // 3a21-PYBV11 ], "console": "integratedTerminal", "env": { diff --git a/src/testbed_micropython/mpstress/README_flakyness.md b/src/testbed_micropython/mpstress/README_flakyness.md index 3c0e62f..8392417 100644 --- a/src/testbed_micropython/mpstress/README_flakyness.md +++ b/src/testbed_micropython/mpstress/README_flakyness.md @@ -1,58 +1,58 @@ # Testresults -### --scenario=NONE --test=RUN_TESTS_ALL +### --stress_scenario==NONE --test=RUN_TESTS_ALL 12 tentacles --> no error! -### --scenario=DUT_ON_OFF --test=RUN_TESTS_ALL +### --stress_scenario==DUT_ON_OFF --test=RUN_TESTS_ALL 12 tentacles --> no error! -### --scenario=INFRA_MPREMOTE --test=RUN_TESTS_BASIC_B_INT_POW +### --stress_scenario==INFRA_MPREMOTE --test=RUN_TESTS_BASIC_B_INT_POW 12 tentacles --> error after 20s - sometimes -### --scenario=INFRA_MPREMOTE --test=RUN_TESTS_BASIC_B_INT_POW --stress-tentacle-count=5 +### --stress_scenario==INFRA_MPREMOTE --test=RUN_TESTS_BASIC_B_INT_POW --stress-tentacle-count=5 5 tentacles --> no error! -### --scenario=SUBPROCESS_INFRA_MPREMOTE --test=RUN_TESTS_ALL +### --stress_scenario==SUBPROCESS_INFRA_MPREMOTE --test=RUN_TESTS_ALL 12 tentacles --> no error! -### --scenario=SUBPROCESS_INFRA_MPREMOTE_C --test=RUN_TESTS_ALL +### --stress_scenario==SUBPROCESS_INFRA_MPREMOTE_C --test=RUN_TESTS_ALL 12 tentacles --> no error! -### --scenario=INFRA_MPREMOTE --test=SERIAL_TEST +### --stress_scenario==INFRA_MPREMOTE --test=SERIAL_TEST 12 tentacles --> error after 3s -### --scenario=INFRA_MPREMOTE --test=SERIAL_TEST --stress-tentacle-count=10 +### --stress_scenario==INFRA_MPREMOTE --test=SERIAL_TEST --stress-tentacle-count=10 10 tentacles --> error after 1.5s, 2s, 8s -### --scenario=INFRA_MPREMOTE --test=SERIAL_TEST --stress-tentacle-count=7 +### --stress_scenario==INFRA_MPREMOTE --test=SERIAL_TEST --stress-tentacle-count=7 7 tentacles --> error after 8s, 14s, 26s -### --scenario=INFRA_MPREMOTE --test=SERIAL_TEST --stress-tentacle-count=6 +### --stress_scenario==INFRA_MPREMOTE --test=SERIAL_TEST --stress-tentacle-count=6 6 tentacles --> no error! -### --scenario=INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE +### --stress_scenario==INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE -`mpstress --scenario=INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE --tentacle=5f2c --micropython-tests=/home/octoprobe/gits/micropython` +`mpstress --stress_scenario==INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE --tentacle=5f2c --micropython-tests=/home/octoprobe/gits/micropython` 12 tentacles --> error after 4s @@ -76,34 +76,34 @@ try reading again: b'' read_duration_s=1.008089s -### --scenario=INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE --stress-tentacle-count=8 +### --stress_scenario==INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE --stress-tentacle-count=8 8 tentacles --> error after 9s, 19s -### --scenario=INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE --stress-tentacle-count=7 +### --stress_scenario==INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE --stress-tentacle-count=7 7 tentacles --> error after 5s, 25s -### --scenario=NONE --test=SIMPLE_SERIAL_WRITE +### --stress_scenario==NONE --test=SIMPLE_SERIAL_WRITE 12 tentacles --> no error -### --scenario=INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE +### --stress_scenario==INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE ==> 5f2c connected to RHS B7 -`mpstress --scenario=INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE --tentacle=5f2c --micropython-tests=/home/octoprobe/gits/micropython` +`mpstress --stress_scenario==INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE --tentacle=5f2c --micropython-tests=/home/octoprobe/gits/micropython` 12 tentacles --> error after 4s ==> 5f2c connected to USB on computer rear -`mpstress --scenario=INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE --tentacle=5f2c --micropython-tests=/home/octoprobe/gits/micropython` +`mpstress --stress_scenario==INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE --tentacle=5f2c --micropython-tests=/home/octoprobe/gits/micropython` 12 tentacles --> error after 120s, 35s diff --git a/src/testbed_micropython/mpstress/cli.py b/src/testbed_micropython/mpstress/cli.py index 2ce6fb9..9b8de89 100644 --- a/src/testbed_micropython/mpstress/cli.py +++ b/src/testbed_micropython/mpstress/cli.py @@ -24,7 +24,7 @@ from .. import constants from ..mptest import util_testrunner -from .util_stress import EnumScenario, StressThread +from .util_stress import EnumStressScenario, StressThread from .util_test_run import EnumTest, run_test logger = logging.getLogger(__file__) @@ -50,7 +50,7 @@ def complete_scenario() -> list[str]: - return sorted([scenario.name for scenario in EnumScenario]) + return sorted([scenario.name for scenario in EnumStressScenario]) def complete_test() -> list[str]: @@ -90,12 +90,12 @@ def stress( help="Use that many tentacles to generate stress. May be less if less tentacles are connected.", ), ] = 99, # noqa: UP007 - scenario: TyperAnnotated[ + stress_scenario: TyperAnnotated[ str, typer.Option( help="Run this stress scenario.", autocompletion=complete_scenario ), - ] = EnumScenario.DUT_ON_OFF, + ] = EnumStressScenario.DUT_ON_OFF, test: TyperAnnotated[ str, typer.Option(help="Use these test arguments.", autocompletion=complete_test), @@ -115,7 +115,7 @@ def stress( micropython_tests=micropython_tests, tentacle=tentacle, stress_tentacle_count=stress_tentacle_count, - scenario=scenario, + stress_scenario=stress_scenario, test=test, ) else: @@ -127,7 +127,7 @@ def stress( connected_tentacle.tentacle_serial_number ), stress_tentacle_count=stress_tentacle_count, - scenario=scenario, + stress_scenario=stress_scenario, test=test, ) @@ -141,10 +141,10 @@ def test_one_tentacle( micropython_tests: str, tentacle: str, stress_tentacle_count: int, - scenario: str, + stress_scenario: str, test: str, ): - scenario = EnumScenario[scenario] + stress_scenario = EnumStressScenario[stress_scenario] tentacle_test: TentacleMicropython | None = None tentacles_load: ConnectedTentacles = ConnectedTentacles() for t in connected_tentacles: @@ -171,7 +171,7 @@ def test_one_tentacle( assert repo_micropython_tests.is_dir(), repo_micropython_tests st = StressThread( - scenario=scenario, + scenario=stress_scenario, stress_tentacle_count=stress_tentacle_count, tentacles_stress=tentacles_load, directory_results=DIRECTORY_RESULTS, diff --git a/src/testbed_micropython/mpstress/util_stress.py b/src/testbed_micropython/mpstress/util_stress.py index 6c6d391..ccc5607 100644 --- a/src/testbed_micropython/mpstress/util_stress.py +++ b/src/testbed_micropython/mpstress/util_stress.py @@ -35,7 +35,7 @@ def print_fds() -> None: print(fd_text.decode("ascii")) -class EnumScenario(enum.StrEnum): +class EnumStressScenario(enum.StrEnum): NONE = enum.auto() DUT_ON_OFF = enum.auto() INFRA_MPREMOTE = enum.auto() @@ -46,12 +46,12 @@ class EnumScenario(enum.StrEnum): class StressThread(threading.Thread): def __init__( self, - scenario: EnumScenario, + scenario: EnumStressScenario, stress_tentacle_count: int, tentacles_stress: ConnectedTentacles, directory_results: pathlib.Path, ): - assert isinstance(scenario, EnumScenario) + assert isinstance(scenario, EnumStressScenario) assert isinstance(stress_tentacle_count, int) assert isinstance(tentacles_stress, ConnectedTentacles) assert isinstance(directory_results, pathlib.Path) @@ -81,19 +81,19 @@ def run(self) -> None: Power up all duts on all tentacles. Now loop over all tentacles and power down dut for a short time """ - if self._scenario is EnumScenario.NONE: + if self._scenario is EnumStressScenario.NONE: return self._scenario_NONE() - if self._scenario is EnumScenario.DUT_ON_OFF: + if self._scenario is EnumStressScenario.DUT_ON_OFF: return self._scenario_DUT_ON_OFF() - if self._scenario is EnumScenario.INFRA_MPREMOTE: + if self._scenario is EnumStressScenario.INFRA_MPREMOTE: return self._scenario_INFRA_MPREMOTE() - if self._scenario is EnumScenario.SUBPROCESS_INFRA_MPREMOTE: + if self._scenario is EnumStressScenario.SUBPROCESS_INFRA_MPREMOTE: return self._scenario_SUBPROCESS_INFRA_MPREMOTE() - if self._scenario is EnumScenario.SUBPROCESS_INFRA_MPREMOTE_C: + if self._scenario is EnumStressScenario.SUBPROCESS_INFRA_MPREMOTE_C: return self._scenario_SUBPROCESS_INFRA_MPREMOTE_C() raise ValueError(f"Not handled: scenario {self._scenario}!") @@ -159,6 +159,8 @@ def _scenario_INFRA_MPREMOTE(self) -> None: for _idx, t in enumerate(self._tentacles_stress): if self._stopping: + for _t in self._tentacles_stress: + _t.infra.mp_remote_close() return i += 1 # if idx > 5: diff --git a/src/testbed_micropython/mpstress/util_test_run.py b/src/testbed_micropython/mpstress/util_test_run.py index 2314062..46fa666 100644 --- a/src/testbed_micropython/mpstress/util_test_run.py +++ b/src/testbed_micropython/mpstress/util_test_run.py @@ -150,11 +150,6 @@ def run_test( time.sleep(1.0) test_args = test.get_test_args(tentacle_test) - # timeout_s = test_params.timeout_s - # for mcu in ("ESP32", "LOLIN_D1", "WB55"): - # if mcu in tentacle_test.description_short: - # timeout_s *= 15 - # logger.info(f"*** {mcu}: timeout_s={timeout_s:0.0f}s") args_aux: list[str] = [] cwd = repo_micropython_tests / MICROPYTHON_DIRECTORY_TESTS From e06df93de4d42c34c2bfbc8165ee536f5313b2e4 Mon Sep 17 00:00:00 2001 From: Hans Maerki Date: Mon, 5 Jan 2026 10:01:35 +0100 Subject: [PATCH 14/15] mpstress tuning --- .../mpstress/README_flakyness.md | 26 +++++++++++++++++++ src/testbed_micropython/mpstress/cli.py | 2 ++ .../mpstress/simple_serial_write.py | 5 +++- .../mpstress/util_stress.py | 2 +- .../mpstress/util_test_run.py | 1 + src/testbed_micropython/mptest/cli.py | 2 +- .../util_subprocess_tentacle.py | 11 +++++--- 7 files changed, 43 insertions(+), 6 deletions(-) diff --git a/src/testbed_micropython/mpstress/README_flakyness.md b/src/testbed_micropython/mpstress/README_flakyness.md index 8392417..53602ce 100644 --- a/src/testbed_micropython/mpstress/README_flakyness.md +++ b/src/testbed_micropython/mpstress/README_flakyness.md @@ -108,3 +108,29 @@ 12 tentacles --> error after 120s, 35s --> no error 340s, 340s + + +## How to run test with ftrace + +* tty_open: https://github.com/torvalds/linux/blob/master/drivers/tty/tty_io.c#L467 + +```bash +sudo chmod a+w /sys/kernel/tracing/trace_marker +cd /tmp/ftrace +sudo trace-cmd record -p function_graph \ + -l tty_open \ + -l tty_poll \ + -l tty_release \ + -l tty_read \ + -l tty_write \ + -l usb_serial_open \ + -l acm_open +``` + +```bash +mpstress --micropython-tests=/home/octoprobe/work_octoprobe/micropython --stress-scenario=NONE --test=SIMPLE_SERIAL_WRITE --stress-tentacle-count=99 2>&1 | tee > ./src/testbed_micropython/mpstress/ftrace/mpstress.log +``` + +```bash +trace-cmd report +``` diff --git a/src/testbed_micropython/mpstress/cli.py b/src/testbed_micropython/mpstress/cli.py index 9b8de89..dd00231 100644 --- a/src/testbed_micropython/mpstress/cli.py +++ b/src/testbed_micropython/mpstress/cli.py @@ -12,6 +12,7 @@ import logging import pathlib +import sys import typer import typing_extensions @@ -102,6 +103,7 @@ def stress( ] = EnumTest.RUN_TESTS_BASIC_B_INT_POW, ) -> None: init_logging() + logger.info(" ".join(sys.argv)) connected_tentacles = util_testrunner.query_connected_tentacles_fast() # connected_tentacles.sort( # key=lambda t: (t.tentacle_spec_base.tentacle_tag, t.tentacle_serial_number) diff --git a/src/testbed_micropython/mpstress/simple_serial_write.py b/src/testbed_micropython/mpstress/simple_serial_write.py index 6c90e4a..5d28931 100644 --- a/src/testbed_micropython/mpstress/simple_serial_write.py +++ b/src/testbed_micropython/mpstress/simple_serial_write.py @@ -122,9 +122,12 @@ def read_test(self, count0: int) -> None: print_fds() raise TestError("Received erronous data.") + count_text = self.serial.read(6) + assert len(count_text) == 6, f"Expected a number of 6 chars: {count_text!r}" count0 = int(count_text) - assert count0 == i0 + assert count0 == i0, f"Unexpected count: {count0} == {i0}" + byte_count += len(CHARS) + 6 if (i1 % 1000) == 0: duration_s = time.monotonic() - start_s diff --git a/src/testbed_micropython/mpstress/util_stress.py b/src/testbed_micropython/mpstress/util_stress.py index ccc5607..efed4ba 100644 --- a/src/testbed_micropython/mpstress/util_stress.py +++ b/src/testbed_micropython/mpstress/util_stress.py @@ -21,7 +21,7 @@ logger = logging.getLogger(__file__) PRINT_STRESS_OUTPUT = False -LOG_OUTPUT_S = 10.0 +LOG_OUTPUT_S = 3.0 # pylint: disable=invalid-name # pylint: disable=no-member diff --git a/src/testbed_micropython/mpstress/util_test_run.py b/src/testbed_micropython/mpstress/util_test_run.py index 46fa666..45cd266 100644 --- a/src/testbed_micropython/mpstress/util_test_run.py +++ b/src/testbed_micropython/mpstress/util_test_run.py @@ -107,6 +107,7 @@ def get_test_args(self, tentacle_test: TentacleMicropython) -> TestArgs: duration_factor = 1 duration_factor = 5 duration_factor = 2 + duration_factor = 4 # duration_factor = 100 count = int(duration_factor * 10000 * serial_speed / serial_speed_default) count = max(1000, count) diff --git a/src/testbed_micropython/mptest/cli.py b/src/testbed_micropython/mptest/cli.py index fdef0a8..797ebad 100644 --- a/src/testbed_micropython/mptest/cli.py +++ b/src/testbed_micropython/mptest/cli.py @@ -390,7 +390,7 @@ def report( envvar="TESTBED_MICROPYTHON_TESTRESULTS", help="Directory containing results", ), - ] = DIRECTORY_TESTRESULTS_DEFAULT, + ] = str(DIRECTORY_TESTRESULTS_DEFAULT), url: TyperAnnotated[ str, typer.Option( diff --git a/src/testbed_micropython/util_subprocess_tentacle.py b/src/testbed_micropython/util_subprocess_tentacle.py index 41866e4..57d5efa 100644 --- a/src/testbed_micropython/util_subprocess_tentacle.py +++ b/src/testbed_micropython/util_subprocess_tentacle.py @@ -76,7 +76,12 @@ def __enter__(self) -> TentaclePowerOffTimer: self._thread.start() return self - def __exit__(self, exc_type:typing.Any, value:typing.Any, traceback:typing.Any) -> None: + def __exit__( + self, + exc_type: typing.Any, + value: typing.Any, + traceback: typing.Any, + ) -> None: # Ensure timer is cancelled and thread is finished self.cancel() @@ -127,7 +132,7 @@ def tentacle_subprocess_run( try: assert logfile is not None logger.info(f"EXEC {args_text}") - logger.info(f"EXEC cwd={cwd}") + logger.info(f"EXEC cwd: {cwd}") logger.info(f"EXEC stdout: {logfile}") logfile.parent.mkdir(parents=True, exist_ok=True) with logfile.open("w") as f: @@ -228,7 +233,7 @@ def sub_run() -> subprocess.CompletedProcess[str]: def log(f: typing.Callable[[str], None]) -> None: f(f"EXEC {args_text}") - f(f" cwd={cwd}") + f(f" cwd: {cwd}") f(f" returncode: {proc.returncode}") f(f" success_codes: {success_returncodes}") f(f" duration: {time.monotonic() - begin_s:0.3f}s") From 85d24bfc70e1aa2ad83f88394e4f6a925380590f Mon Sep 17 00:00:00 2001 From: Hans Maerki Date: Sun, 11 Jan 2026 15:30:06 +0100 Subject: [PATCH 15/15] tmp --- .vscode/launch.json | 14 ++++++++------ .../mpstress/simple_serial_write.py | 6 +----- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 12ed184..3b740f1 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -135,24 +135,25 @@ // "--only-test=RUN-TESTS_STANDARD", // "--only-test=RUN-TESTS_STANDARD_NATIVE", // "--only-test=RUN-TESTS_STANDARD_VIA_MPY", - "--only-test=RUN-TESTS_EXTMOD_HARDWARE", + // "--only-test=RUN-TESTS_EXTMOD_HARDWARE", // "--only-test=RUN-FLASH_FORMAT", // "--only-test=RUN-TESTS_STANDARD_NATIVE", // "--only-test=RUN-TESTS_STANDARD", // "--skip-test=RUN-TESTS_STANDARD", // "--only-test=RUN-TESTS_STANDARD:run-tests.py --via-mpy --test-dirs=micropython", // "--only-test=RUN-TESTS_STANDARD:run-tests.py --test-dirs=micropython", + "--only-test=RUN-TESTS_STANDARD:run-tests.py --test-dirs=dummy", // "--only-board=LOLIN_D1_MINI", // "--skip-test=RUN-MULTITESTS_MULTINET", // "--only-test=RUN-NATMODTESTS", // "--only-board=LOLIN_C3_MINI", - "--only-board=LOLIN_D1_MINI", + // "--only-board=LOLIN_D1_MINI", // "--only-fut=FUT_WLAN", // "--skip-fut=FUT_WLAN", // "--skip-fut=FUT_BLE", "--no-multiprocessing", // "--flash-force", - // "--flash-skip", + "--flash-skip", // "--count=24", // "--no-git-clean", // "--debug-skip-tests", @@ -224,9 +225,9 @@ // "stress", // "--micropython-tests=${workspaceFolder}/../fork_micropython", "--micropython-tests=${workspaceFolder}/../micropython", - // "--stress-scenario=NONE", + "--stress-scenario=NONE", // "--stress-scenario=DUT_ON_OFF", - "--stress-scenario=INFRA_MPREMOTE", + // "--stress-scenario=INFRA_MPREMOTE", // "--stress-scenario=SUBPROCESS_INFRA_MPREMOTE", // "--stress-scenario=SUBPROCESS_INFRA_MPREMOTE_C", // "--test=RUN_TESTS_BASIC_B_INT_POW", @@ -244,9 +245,10 @@ "console": "integratedTerminal", "env": { "PYDEVD_DISABLE_FILE_VALIDATION": "1", + "OCTOPROBE_ENABLE_FTRACE_MARKER": "1" }, "justMyCode": false, - "subProcess": true, + "subProcess": false, }, { "name": "simple_serial_write", diff --git a/src/testbed_micropython/mpstress/simple_serial_write.py b/src/testbed_micropython/mpstress/simple_serial_write.py index 5d28931..4be37e9 100644 --- a/src/testbed_micropython/mpstress/simple_serial_write.py +++ b/src/testbed_micropython/mpstress/simple_serial_write.py @@ -35,11 +35,7 @@ CHARS = b"_ABCDEFGHIJKLMNOPQRSTUVWXYabcdefghijklmnopqrstuvwxy1234567890_" def write(msg): - try: - sys.stdout.buffer.write(msg) - except AttributeError: - # 2d2d_LOLIN_D1_MINI - print(msg.decode("ascii"), end="") + sys.stdout.write(msg) def send_alphabet(count): for i in range(count):