diff --git a/.gitignore b/.gitignore index e70f445..f10127a 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,5 @@ src/testbed_micropython/mpbuild/tests/results/*.txt src/testbed_micropython/mpbuild/docker-build-micropython-esp32/esp-idf src/testbed/experiments/* testresults*/** +src/testbed_micropython/mpstress/testresults*/** +src/testbed_micropython/mpstress/c/mpremote_c diff --git a/.vscode/launch.json b/.vscode/launch.json index 935d7b5..3b740f1 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -135,13 +135,14 @@ // "--only-test=RUN-TESTS_STANDARD", // "--only-test=RUN-TESTS_STANDARD_NATIVE", // "--only-test=RUN-TESTS_STANDARD_VIA_MPY", - "--only-test=RUN-TESTS_EXTMOD_HARDWARE", + // "--only-test=RUN-TESTS_EXTMOD_HARDWARE", // "--only-test=RUN-FLASH_FORMAT", // "--only-test=RUN-TESTS_STANDARD_NATIVE", // "--only-test=RUN-TESTS_STANDARD", // "--skip-test=RUN-TESTS_STANDARD", // "--only-test=RUN-TESTS_STANDARD:run-tests.py --via-mpy --test-dirs=micropython", // "--only-test=RUN-TESTS_STANDARD:run-tests.py --test-dirs=micropython", + "--only-test=RUN-TESTS_STANDARD:run-tests.py --test-dirs=dummy", // "--only-board=LOLIN_D1_MINI", // "--skip-test=RUN-MULTITESTS_MULTINET", // "--only-test=RUN-NATMODTESTS", @@ -213,6 +214,59 @@ "justMyCode": false, "subProcess": false, }, + { + "name": "mpstress", + "type": "debugpy", + "request": "launch", + "module": "testbed_micropython.mpstress.cli", + "cwd": "${workspaceFolder}", + "args": [ + // "--help", + // "stress", + // "--micropython-tests=${workspaceFolder}/../fork_micropython", + "--micropython-tests=${workspaceFolder}/../micropython", + "--stress-scenario=NONE", + // "--stress-scenario=DUT_ON_OFF", + // "--stress-scenario=INFRA_MPREMOTE", + // "--stress-scenario=SUBPROCESS_INFRA_MPREMOTE", + // "--stress-scenario=SUBPROCESS_INFRA_MPREMOTE_C", + // "--test=RUN_TESTS_BASIC_B_INT_POW", + // "--test=SERIAL_TEST", + "--test=SIMPLE_SERIAL_WRITE", + "--stress-tentacle-count=99", + // "--tentacle=5f2a", // 5f2a-ADA_ITSYBITSY_M0 + // "--tentacle=5f2c", // 5f2c-RPI_PICO_W + // "--tentacle=3c2a", // 3c2a-ARDUINO_NANO_33 + // "--tentacle=0c30", // 0c30-ESP32_C3_DEVKIT + // "--tentacle=5d21", // 5d21-ESP32_DEVKIT + // "--tentacle=2d2d", // 2d2d_LOLIN_D1_MINI + // "--tentacle=3a21", // 3a21-PYBV11 + ], + "console": "integratedTerminal", + "env": { + "PYDEVD_DISABLE_FILE_VALIDATION": "1", + "OCTOPROBE_ENABLE_FTRACE_MARKER": "1" + }, + "justMyCode": false, + "subProcess": false, + }, + { + "name": "simple_serial_write", + "type": "debugpy", + "request": "launch", + "module": "testbed_micropython.mpstress.simple_serial_write", + "cwd": "${workspaceFolder}", + "args": [ + "--count=10000", + "--test-instance=port:/dev/ttyUSB0" + ], + "console": "integratedTerminal", + "env": { + "PYDEVD_DISABLE_FILE_VALIDATION": "1", + }, + "justMyCode": false, + "subProcess": false, + }, { "name": "pytest", "type": "debugpy", diff --git a/pyproject.toml b/pyproject.toml index 443b84d..1792c82 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ Repository = "https://github.com/octoprobe/octoprobe" [project.scripts] mptest = "testbed_micropython.mptest.cli:app" +mpstress = "testbed_micropython.mpstress.cli:app" [project.optional-dependencies] diff --git a/src/testbed_micropython/mpstress/README_flakyness.md b/src/testbed_micropython/mpstress/README_flakyness.md new file mode 100644 index 0000000..53602ce --- /dev/null +++ b/src/testbed_micropython/mpstress/README_flakyness.md @@ -0,0 +1,136 @@ +# Testresults + +### --stress_scenario==NONE --test=RUN_TESTS_ALL + +12 tentacles +--> no error! + +### --stress_scenario==DUT_ON_OFF --test=RUN_TESTS_ALL + +12 tentacles +--> no error! + +### --stress_scenario==INFRA_MPREMOTE --test=RUN_TESTS_BASIC_B_INT_POW + +12 tentacles +--> error after 20s - sometimes + +### --stress_scenario==INFRA_MPREMOTE --test=RUN_TESTS_BASIC_B_INT_POW --stress-tentacle-count=5 + +5 tentacles +--> no error! + +### --stress_scenario==SUBPROCESS_INFRA_MPREMOTE --test=RUN_TESTS_ALL + +12 tentacles +--> no error! + +### --stress_scenario==SUBPROCESS_INFRA_MPREMOTE_C --test=RUN_TESTS_ALL + +12 tentacles +--> no error! + +### --stress_scenario==INFRA_MPREMOTE --test=SERIAL_TEST + +12 tentacles +--> error after 3s + +### --stress_scenario==INFRA_MPREMOTE --test=SERIAL_TEST --stress-tentacle-count=10 + +10 tentacles +--> error after 1.5s, 2s, 8s + +### --stress_scenario==INFRA_MPREMOTE --test=SERIAL_TEST --stress-tentacle-count=7 + +7 tentacles +--> error after 8s, 14s, 26s + +### --stress_scenario==INFRA_MPREMOTE --test=SERIAL_TEST --stress-tentacle-count=6 + +6 tentacles +--> no error! + +### --stress_scenario==INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE + +`mpstress --stress_scenario==INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE --tentacle=5f2c --micropython-tests=/home/octoprobe/gits/micropython` + +12 tentacles +--> error after 4s + 006000: 197kBytes/s + ERROR, read_duration_s=1.001166s + expected: b'_ABCDEFGHIJKLMNOPQRSTUVWXYabcdefghijklmnopqrstuvwxy1234567890_' + received: b'_ABCDEFGHIJKLMNOPQRSTUVWXYabcdefghijklmnopqrstuvwxy123456789' + try reading again: b'' read_duration_s=1.001263s + + # Debug output from serialposix.py: + read(6(6)) duration=0.000011s + [4] = select(1.000s) duration=0.000018s + read(44(62)) duration=0.000010s + [] = select(1.000s) duration=1.000669s + TIMEOUT! + ERROR, read_duration_s=1.000970s + expected: b'_ABCDEFGHIJKLMNOPQRSTUVWXYabcdefghijklmnopqrstuvwxy1234567890_' + received: b'_ABCDEFGHIJKLMNOPQRSTUVWXYabcdefghijklmnopqr' + [] = select(1.000s) duration=1.000536s + TIMEOUT! + try reading again: b'' read_duration_s=1.008089s + + +### --stress_scenario==INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE --stress-tentacle-count=8 + +8 tentacles +--> error after 9s, 19s + +### --stress_scenario==INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE --stress-tentacle-count=7 + +7 tentacles +--> error after 5s, 25s + +### --stress_scenario==NONE --test=SIMPLE_SERIAL_WRITE + +12 tentacles +--> no error + + + +### --stress_scenario==INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE + +==> 5f2c connected to RHS B7 +`mpstress --stress_scenario==INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE --tentacle=5f2c --micropython-tests=/home/octoprobe/gits/micropython` + +12 tentacles +--> error after 4s + +==> 5f2c connected to USB on computer rear + +`mpstress --stress_scenario==INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE --tentacle=5f2c --micropython-tests=/home/octoprobe/gits/micropython` + +12 tentacles +--> error after 120s, 35s +--> no error 340s, 340s + + +## How to run test with ftrace + +* tty_open: https://github.com/torvalds/linux/blob/master/drivers/tty/tty_io.c#L467 + +```bash +sudo chmod a+w /sys/kernel/tracing/trace_marker +cd /tmp/ftrace +sudo trace-cmd record -p function_graph \ + -l tty_open \ + -l tty_poll \ + -l tty_release \ + -l tty_read \ + -l tty_write \ + -l usb_serial_open \ + -l acm_open +``` + +```bash +mpstress --micropython-tests=/home/octoprobe/work_octoprobe/micropython --stress-scenario=NONE --test=SIMPLE_SERIAL_WRITE --stress-tentacle-count=99 2>&1 | tee > ./src/testbed_micropython/mpstress/ftrace/mpstress.log +``` + +```bash +trace-cmd report +``` diff --git a/src/testbed_micropython/mpstress/__init__.py b/src/testbed_micropython/mpstress/__init__.py new file mode 100644 index 0000000..b1a19e3 --- /dev/null +++ b/src/testbed_micropython/mpstress/__init__.py @@ -0,0 +1 @@ +__version__ = "0.0.5" diff --git a/src/testbed_micropython/mpstress/c/Makefile b/src/testbed_micropython/mpstress/c/Makefile new file mode 100644 index 0000000..a0190df --- /dev/null +++ b/src/testbed_micropython/mpstress/c/Makefile @@ -0,0 +1,12 @@ +CC=gcc +CFLAGS=-Wall -Wextra -std=c99 +TARGET=mpremote_c +SOURCE=mpremote_c.c + +$(TARGET): $(SOURCE) + $(CC) $(CFLAGS) -o $(TARGET) $(SOURCE) + +clean: + rm -f $(TARGET) + +.PHONY: clean \ No newline at end of file diff --git a/src/testbed_micropython/mpstress/c/mpremote_c.c b/src/testbed_micropython/mpstress/c/mpremote_c.c new file mode 100644 index 0000000..4871a51 --- /dev/null +++ b/src/testbed_micropython/mpstress/c/mpremote_c.c @@ -0,0 +1,147 @@ +/* +This programs open a serial line to a micropython cpu. + +Send "print('hello')" +Return 0 if the response in "hellon\r\n" +Return 1 on error. +*/ +#define _DEFAULT_SOURCE +#define _BSD_SOURCE +#include +#include +#include +#include +#include +#include +#include + +#ifndef CRTSCTS +#define CRTSCTS 0 +#endif + +int main(int argc, char *argv[]) { + if (argc != 2) { + fprintf(stderr, "Usage: %s \n", argv[0]); + fprintf(stderr, "Example: %s /dev/ttyACM4\n", argv[0]); + return 1; + } + + const char *serial_device = argv[1]; + int serial_fd; + struct termios tty; + const char *command = "print('hello')\r\n"; + char response[256]; + const char *expected_response = "hello\r\n"; + ssize_t bytes_read; + + // Open serial device + serial_fd = open(serial_device, O_RDWR | O_NOCTTY); + if (serial_fd < 0) { + perror("Error opening serial device"); + return 1; + } + + // Get current terminal attributes + if (tcgetattr(serial_fd, &tty) != 0) { + perror("Error getting terminal attributes"); + close(serial_fd); + return 1; + } + + // Configure serial port settings + // Set baud rate to 115200 + cfsetospeed(&tty, B115200); + cfsetispeed(&tty, B115200); + + // 8N1 mode + tty.c_cflag &= ~PARENB; // No parity + tty.c_cflag &= ~CSTOPB; // One stop bit + tty.c_cflag &= ~CSIZE; // Clear size bits + tty.c_cflag |= CS8; // 8 data bits + tty.c_cflag &= ~CRTSCTS; // No hardware flow control + tty.c_cflag |= CREAD | CLOCAL; // Enable reading, ignore modem control lines + + // Input modes + tty.c_iflag &= ~(IXON | IXOFF | IXANY); // No software flow control + tty.c_iflag &= ~(ICANON | ECHO | ECHOE | ISIG); // Raw input + + // Output modes + tty.c_oflag &= ~OPOST; // Raw output + + // Local modes + tty.c_lflag &= ~(ICANON | ECHO | ECHOE | ISIG); // Raw mode + + // Control characters + tty.c_cc[VMIN] = 2*strlen(expected_response); // Blocking read - wait for at least 1 character + tty.c_cc[VTIME] = 10; // 1 second timeout (in deciseconds) + + // Apply settings + if (tcsetattr(serial_fd, TCSANOW, &tty) != 0) { + perror("Error setting terminal attributes"); + close(serial_fd); + return 1; + } + + // Flush any existing data + tcflush(serial_fd, TCIOFLUSH); + + // Give device time to initialize + // usleep(100000); // 100ms + + // Send command + ssize_t bytes_written = write(serial_fd, command, strlen(command)); + if (bytes_written < 0) { + perror("Error writing to serial device"); + close(serial_fd); + return 1; + } + + // Give device time to process and respond + // usleep(500000); // 500ms + + // Read response + memset(response, 0, sizeof(response)); + bytes_read = read(serial_fd, response, sizeof(response) - 1); + if (bytes_read < 0) { + perror("Error reading from serial device"); + close(serial_fd); + return 1; + } + + // Close serial device + close(serial_fd); + + // Check if we got the expected response + // Look for "hello" followed by newline and carriage return + if (strstr(response, "hello") != NULL) { + // Check if response contains hello followed by \r\n or \n\r + if (strstr(response, "hello\r\n") != NULL || strstr(response, "hello\n\r") != NULL) { + printf("Success: Got expected response containing 'hello\\r\\n' or 'hello\\n\\r'\n"); + return 0; + } else if (strstr(response, "hello") != NULL) { + printf("Partial success: Got 'hello' but not with expected line endings\n"); + printf("Response was: "); + for (int i = 0; i < bytes_read; i++) { + if (response[i] >= 32 && response[i] <= 126) { + printf("%c", response[i]); + } else { + printf("\\x%02x", (unsigned char)response[i]); + } + } + printf("\n"); + return 0; // Still consider it success if we got hello + } + } + + printf("Failed: Did not get expected response 'hello\\n\\r'\n"); + printf("Response was: "); + for (int i = 0; i < bytes_read; i++) { + if (response[i] >= 32 && response[i] <= 126) { + printf("%c", response[i]); + } else { + printf("\\x%02x", (unsigned char)response[i]); + } + } + printf("\n"); + return 1; +} diff --git a/src/testbed_micropython/mpstress/cli.py b/src/testbed_micropython/mpstress/cli.py new file mode 100644 index 0000000..dd00231 --- /dev/null +++ b/src/testbed_micropython/mpstress/cli.py @@ -0,0 +1,196 @@ +""" +Where to take the tests from +--micropython-tests-giturl=https://github.com/dpgeorge/micropython.git@tests-full-test-runner + +Where to take the firmware from +--firmware-build-giturl=https://github.com/micropython/micropython.git@v1.24.1 +--firmware-build-gitdir=~/micropython +--firmware-gitdir=~/micropython +""" + +from __future__ import annotations + +import logging +import pathlib +import sys + +import typer +import typing_extensions +from octoprobe import util_baseclasses +from octoprobe.scripts.commissioning import init_logging +from octoprobe.usb_tentacle.usb_tentacle import serial_short_from_delimited + +from testbed_micropython.tentacle_spec import TentacleMicropython +from testbed_micropython.testcollection.baseclasses_spec import ConnectedTentacles + +from .. import constants +from ..mptest import util_testrunner +from .util_stress import EnumStressScenario, StressThread +from .util_test_run import EnumTest, run_test + +logger = logging.getLogger(__file__) + +# 'typer' does not work correctly with typing.Annotated +# Required is: typing_extensions.Annotated +TyperAnnotated = typing_extensions.Annotated + +# mypy: disable-error-code="valid-type" +# This will disable this warning: +# op.py:58: error: Variable "octoprobe.scripts.op.TyperAnnotated" is not valid as a type [valid-type] +# op.py:58: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases + +app = typer.Typer(pretty_exceptions_enable=False) + +DIRECTORY_OF_THIS_FILE = pathlib.Path(__file__).parent +DIRECTORY_RESULTS = DIRECTORY_OF_THIS_FILE / "testresults" +DIRECTORY_RESULTS.mkdir(parents=True, exist_ok=True) +for f in DIRECTORY_RESULTS.glob("*.txt"): + f.unlink(missing_ok=True) +for f in DIRECTORY_RESULTS.glob("*.out"): + f.unlink(missing_ok=True) + + +def complete_scenario() -> list[str]: + return sorted([scenario.name for scenario in EnumStressScenario]) + + +def complete_test() -> list[str]: + return sorted([test.name for test in EnumTest]) + + +def complete_only_tentacle() -> list[str]: + connected_tentacles = util_testrunner.query_connected_tentacles_fast() + + return sorted( + { + serial_short_from_delimited(t.tentacle_instance.serial) + for t in connected_tentacles + } + ) + + +@app.command(help="Put load on all tentacles to provoke stress") +def stress( + micropython_tests: TyperAnnotated[ + str, + typer.Option( + envvar="TESTBED_MICROPYTHON_MICROPYTHON_TESTS", + help="Directory of MicroPython-Repo with the tests. Example ~/micropython or https://github.com/micropython/micropython.git@v1.24.1", + ), + ] = constants.URL_FILENAME_DEFAULT, + tentacle: TyperAnnotated[ + str | None, + typer.Option( + help="Run tests only on these tentacles. All other tentacles are used to create stress", + autocompletion=complete_only_tentacle, + ), + ] = None, # noqa: UP007 + stress_tentacle_count: TyperAnnotated[ + int, + typer.Option( + help="Use that many tentacles to generate stress. May be less if less tentacles are connected.", + ), + ] = 99, # noqa: UP007 + stress_scenario: TyperAnnotated[ + str, + typer.Option( + help="Run this stress scenario.", autocompletion=complete_scenario + ), + ] = EnumStressScenario.DUT_ON_OFF, + test: TyperAnnotated[ + str, + typer.Option(help="Use these test arguments.", autocompletion=complete_test), + ] = EnumTest.RUN_TESTS_BASIC_B_INT_POW, +) -> None: + init_logging() + logger.info(" ".join(sys.argv)) + connected_tentacles = util_testrunner.query_connected_tentacles_fast() + # connected_tentacles.sort( + # key=lambda t: (t.tentacle_spec_base.tentacle_tag, t.tentacle_serial_number) + # ) + connected_tentacles.sort(key=lambda t: t.tentacle_serial_number) + + try: + if tentacle is not None: + test_one_tentacle( + connected_tentacles=connected_tentacles, + micropython_tests=micropython_tests, + tentacle=tentacle, + stress_tentacle_count=stress_tentacle_count, + stress_scenario=stress_scenario, + test=test, + ) + else: + for connected_tentacle in connected_tentacles: + test_one_tentacle( + connected_tentacles=connected_tentacles, + micropython_tests=micropython_tests, + tentacle=serial_short_from_delimited( + connected_tentacle.tentacle_serial_number + ), + stress_tentacle_count=stress_tentacle_count, + stress_scenario=stress_scenario, + test=test, + ) + + except util_baseclasses.OctoprobeAppExitException as e: + logger.info(f"Terminating test due to OctoprobeAppExitException: {e}") + raise typer.Exit(1) from e + + +def test_one_tentacle( + connected_tentacles: ConnectedTentacles, + micropython_tests: str, + tentacle: str, + stress_tentacle_count: int, + stress_scenario: str, + test: str, +): + stress_scenario = EnumStressScenario[stress_scenario] + tentacle_test: TentacleMicropython | None = None + tentacles_load: ConnectedTentacles = ConnectedTentacles() + for t in connected_tentacles: + serial_short = serial_short_from_delimited(t.tentacle_serial_number) + if serial_short == tentacle: # type: ignore[attr-defined] + tentacle_test = t + continue + tentacles_load.append(t) + + assert tentacle_test is not None, f"Tentacle not connected: {tentacle}" + print(f"*** Initialized {len(connected_tentacles)} tentacles") + for t in connected_tentacles: + print(f"**** load_base_code_if_needed {t.description_short}") + t.infra.load_base_code_if_needed() + t.switches.default_off_infra_on() + # if scenario is EnumScenario.INFRA_MPREMOTE: + # # if t == tentacle_test: + # # t.infra.switches.dut = True + # # else: + # # t.infra.switches.dut = False + # t.infra.switches.dut = False + + repo_micropython_tests = pathlib.Path(micropython_tests).expanduser().resolve() + assert repo_micropython_tests.is_dir(), repo_micropython_tests + + st = StressThread( + scenario=stress_scenario, + stress_tentacle_count=stress_tentacle_count, + tentacles_stress=tentacles_load, + directory_results=DIRECTORY_RESULTS, + ) + + print("*** start") + st.start() + print("*** run_test") + run_test( + tentacle_test=tentacle_test, + repo_micropython_tests=repo_micropython_tests, + directory_results=DIRECTORY_RESULTS, + test=EnumTest[test], + ) + print("*** stop") + st.stop() + + +if __name__ == "__main__": + app() diff --git a/src/testbed_micropython/mpstress/simple_serial_write.py b/src/testbed_micropython/mpstress/simple_serial_write.py new file mode 100644 index 0000000..4be37e9 --- /dev/null +++ b/src/testbed_micropython/mpstress/simple_serial_write.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +import dataclasses +import logging +import os +import time + +import serial +import typer +import typing_extensions + +from testbed_micropython.mpstress.util_stress import print_fds + +logger = logging.getLogger(__file__) + +# 'typer' does not work correctly with typing.Annotated +# Required is: typing_extensions.Annotated +TyperAnnotated = typing_extensions.Annotated + +# mypy: disable-error-code="valid-type" +# This will disable this warning: +# op.py:58: error: Variable "octoprobe.scripts.op.TyperAnnotated" is not valid as a type [valid-type] +# op.py:58: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases + +app = typer.Typer(pretty_exceptions_enable=False) + +CHARS = b"_ABCDEFGHIJKLMNOPQRSTUVWXYabcdefghijklmnopqrstuvwxy1234567890_" + +MICROPYTHON_SCRIPT = """ +# Based on +# /tests/serial_test.py: read_test_script + +import sys + +CHARS = b"_ABCDEFGHIJKLMNOPQRSTUVWXYabcdefghijklmnopqrstuvwxy1234567890_" + +def write(msg): + sys.stdout.write(msg) + +def send_alphabet(count): + for i in range(count): + write(CHARS) + # 2d2d_LOLIN_D1_MINI + # sys.stdout.buffer.write(f"{i:06d}") + write(b"%06d" % i) + +send_alphabet(count=) +""" + + +class TestError(Exception): + pass + + +@dataclasses.dataclass(repr=True, frozen=True) +class SimpleSerialWrite: + serial: serial.Serial + + def drain_input(self) -> None: + time.sleep(0.1) + while self.serial.inWaiting() > 0: # type: ignore[attr-defined] + _data = self.serial.read(self.serial.inWaiting()) # type: ignore[attr-defined] + time.sleep(0.1) + + def send_script(self, script: bytes) -> None: + assert isinstance(script, bytes) + chunk_size = 32 + for i in range(0, len(script), chunk_size): + self.serial.write(script[i : i + chunk_size]) + time.sleep(0.01) + self.serial.write(b"\x04") # eof + self.serial.flush() + response = self.serial.read(2) + if response != b"OK": + response += self.serial.read(self.serial.inWaiting()) # type: ignore[attr-defined] + raise TestError("could not send script", response) + + def read_test(self, count0: int) -> None: + self.serial.write(b"\x03\x01\x04") # break, raw-repl, soft-reboot + self.drain_input() + script = MICROPYTHON_SCRIPT.replace("", str(count0)) + self.send_script(script.encode("ascii")) + + start_s = time.monotonic() + byte_count = 0 + for i0 in range(count0): + i1 = i0 + 1 + read_start_s = time.monotonic() + chars = self.serial.read(len(CHARS)) + read_duration_s = time.monotonic() - read_start_s + if chars != CHARS: + print(f"ERROR, read_duration_s={read_duration_s:0.6f}s") + print(" expected:", CHARS) + print(" received:", chars) + read_start_s = time.monotonic() + chars2 = self.serial.read(len(CHARS)) + read_duration_s = time.monotonic() - read_start_s + print( + f" try reading again: {chars2.decode('ascii')} read_duration_s={read_duration_s:0.6f}s" + ) + print(f" serial:{self.serial!r}") + elements = [ + f"{self.serial.fd=}", + f"{self.serial.in_waiting=}", + f"{self.serial.pipe_abort_read_r=}", + f"{self.serial.pipe_abort_read_w=}", + f"{self.serial.pipe_abort_write_r=}", + f"{self.serial.pipe_abort_write_w=}", + f"{self.serial._rts_state=}", # type: ignore[attr-defined] + f"{self.serial._break_state=}", # type: ignore[attr-defined] + f"{self.serial._dtr_state=}", # type: ignore[attr-defined] + f"{self.serial._dsrdtr=}", # type: ignore[attr-defined] + ] + print(" ", " ".join(elements)) + assert isinstance(self.serial.fd, int), self.serial.fd + print(f" {os.fstat(self.serial.fd)=!r}") + + print_fds() + + raise TestError("Received erronous data.") + + count_text = self.serial.read(6) + assert len(count_text) == 6, f"Expected a number of 6 chars: {count_text!r}" + count0 = int(count_text) + assert count0 == i0, f"Unexpected count: {count0} == {i0}" + + byte_count += len(CHARS) + 6 + if (i1 % 1000) == 0: + duration_s = time.monotonic() - start_s + # print(i, chars, count, f"{byte_count / duration_s / 1_000:0.6f}kBytes/s") + print(f"{i1:06d}: {byte_count / duration_s / 1_000:03.0f}kBytes/s") + start_s = time.monotonic() + byte_count = 0 + + +@app.command(help="Write datastream from micropython ") +def write_alphabet( + test_instance: TyperAnnotated[ + str, + typer.Option(help="For example port:/dev/ttyACM3"), + ], + count: TyperAnnotated[ + int, + typer.Option(help="The count of ~60byte-strings to be sent"), + ] = 10_000, # noqa: UP007 +) -> None: + assert isinstance(test_instance, str) + assert str(test_instance).startswith("port:"), test_instance + + serial_port = test_instance[len("port:") :] # type: ignore[index] + ssw = SimpleSerialWrite(serial.Serial(serial_port, baudrate=115200, timeout=5)) + ssw.read_test(count0=count) + + +if __name__ == "__main__": + app() diff --git a/src/testbed_micropython/mpstress/util_stress.py b/src/testbed_micropython/mpstress/util_stress.py new file mode 100644 index 0000000..efed4ba --- /dev/null +++ b/src/testbed_micropython/mpstress/util_stress.py @@ -0,0 +1,209 @@ +from __future__ import annotations + +import enum +import logging +import os +import pathlib +import subprocess +import sys +import threading +import time + +from octoprobe.usb_tentacle.usb_tentacle import serial_short_from_delimited +from octoprobe.util_subprocess import subprocess_run + +from testbed_micropython.testcollection.baseclasses_spec import ConnectedTentacles + +from ..testcollection.constants import ENV_PYTHONUNBUFFERED + +DIRECTORY_OF_THIS_FILE = pathlib.Path(__file__).parent + +logger = logging.getLogger(__file__) + +PRINT_STRESS_OUTPUT = False +LOG_OUTPUT_S = 3.0 + +# pylint: disable=invalid-name +# pylint: disable=no-member +# pylint: disable=protected-access + + +def print_fds() -> None: + cmd = f"ls -l /proc/{os.getpid()}/fd" + fd_text = subprocess.check_output(cmd, shell=True) + print(f" {cmd}: {len(fd_text.splitlines())} lines") + print(fd_text.decode("ascii")) + + +class EnumStressScenario(enum.StrEnum): + NONE = enum.auto() + DUT_ON_OFF = enum.auto() + INFRA_MPREMOTE = enum.auto() + SUBPROCESS_INFRA_MPREMOTE = enum.auto() + SUBPROCESS_INFRA_MPREMOTE_C = enum.auto() + + +class StressThread(threading.Thread): + def __init__( + self, + scenario: EnumStressScenario, + stress_tentacle_count: int, + tentacles_stress: ConnectedTentacles, + directory_results: pathlib.Path, + ): + assert isinstance(scenario, EnumStressScenario) + assert isinstance(stress_tentacle_count, int) + assert isinstance(tentacles_stress, ConnectedTentacles) + assert isinstance(directory_results, pathlib.Path) + super().__init__(daemon=True, name="stress") + self._next_log_output_s = time.monotonic() + LOG_OUTPUT_S + self._stopping = False + self._stress_tentacle_count = stress_tentacle_count + self._scenario = scenario + self._directory_results = directory_results + print( + f"Found {len(tentacles_stress)} tentacle to create stress. stress_tentacle_count={self._stress_tentacle_count}." + ) + self._tentacles_stress = tentacles_stress[: self._stress_tentacle_count] + print( + f"Tentacles to generate stress: {[serial_short_from_delimited(t.tentacle_instance.serial) for t in self._tentacles_stress]}" + ) + + @property + def do_log_output(self) -> bool: + if time.monotonic() > self._next_log_output_s: + self._next_log_output_s += LOG_OUTPUT_S + return True + return False + + def run(self) -> None: + """ + Power up all duts on all tentacles. + Now loop over all tentacles and power down dut for a short time + """ + if self._scenario is EnumStressScenario.NONE: + return self._scenario_NONE() + + if self._scenario is EnumStressScenario.DUT_ON_OFF: + return self._scenario_DUT_ON_OFF() + + if self._scenario is EnumStressScenario.INFRA_MPREMOTE: + return self._scenario_INFRA_MPREMOTE() + + if self._scenario is EnumStressScenario.SUBPROCESS_INFRA_MPREMOTE: + return self._scenario_SUBPROCESS_INFRA_MPREMOTE() + + if self._scenario is EnumStressScenario.SUBPROCESS_INFRA_MPREMOTE_C: + return self._scenario_SUBPROCESS_INFRA_MPREMOTE_C() + + raise ValueError(f"Not handled: scenario {self._scenario}!") + + def _scenario_NONE(self) -> None: + return + + def _scenario_SUBPROCESS_INFRA_MPREMOTE_C(self) -> None: + i = 0 + while True: + print("cycle") + for t in self._tentacles_stress: + if self._stopping: + return + i += 1 + assert t.infra.usb_tentacle.serial_port is not None + args = [ + str(DIRECTORY_OF_THIS_FILE / "c" / "mpremote_c"), + t.infra.usb_tentacle.serial_port, + ] + env = ENV_PYTHONUNBUFFERED + subprocess_run( + args=args, + cwd=self._directory_results, + env=env, + logfile=self._directory_results / f"mpremote_c_{i:04d}.txt", + timeout_s=1.0, + ) + + def _scenario_SUBPROCESS_INFRA_MPREMOTE(self) -> None: + i = 0 + while True: + print("cycle") + for t in self._tentacles_stress: + if self._stopping: + return + i += 1 + assert t.infra.usb_tentacle.serial_port is not None + args = [ + sys.executable, + "-m", + "mpremote", + "connect", + t.infra.usb_tentacle.serial_port, + "eval", + "print('Hello MicroPython')", + ] + env = ENV_PYTHONUNBUFFERED + subprocess_run( + args=args, + cwd=self._directory_results, + env=env, + logfile=self._directory_results / f"mpremote_{i:04d}.txt", + timeout_s=5.0, + ) + + def _scenario_INFRA_MPREMOTE(self) -> None: + i = 0 + while True: + if PRINT_STRESS_OUTPUT: + print("cycle") + print_fds() + + for _idx, t in enumerate(self._tentacles_stress): + if self._stopping: + for _t in self._tentacles_stress: + _t.infra.mp_remote_close() + return + i += 1 + # if idx > 5: + # continue + serial_closed = t.infra.mp_remote_close() + t.infra.connect_mpremote_if_needed() + assert t.infra._mp_remote is not None + assert t.infra._mp_remote.state.transport is not None + serial = t.infra._mp_remote.state.transport.serial + fds = ( + serial.fd, + serial.pipe_abort_read_r, + serial.pipe_abort_read_w, + serial.pipe_abort_write_r, + serial.pipe_abort_write_w, + ) + # if PRINT_STRESS_OUTPUT: + if self.do_log_output: + print( + f"count={i:03d}", + f"pyserial.fds:{fds}", + f"close:{serial_closed}", + f"open:{t.infra.mp_remote._tty}", + ) + if False: + rc = t.infra.mp_remote.exec_raw("print('Hello')") + assert rc == "Hello\r\n" + time.sleep(0.001) + # print(rc) + + def _scenario_DUT_ON_OFF(self) -> None: + sleep_s = 1.0 + while not self._stopping: + print("on") + for t in self._tentacles_stress: + t.infra.switches.dut = True + time.sleep(sleep_s) + + print("off") + for t in self._tentacles_stress: + t.infra.switches.dut = False + time.sleep(sleep_s) + + def stop(self) -> None: + self._stopping = True + self.join() diff --git a/src/testbed_micropython/mpstress/util_test_run.py b/src/testbed_micropython/mpstress/util_test_run.py new file mode 100644 index 0000000..45cd266 --- /dev/null +++ b/src/testbed_micropython/mpstress/util_test_run.py @@ -0,0 +1,190 @@ +from __future__ import annotations + +import dataclasses +import enum +import logging +import pathlib +import sys +import time + +from octoprobe.util_pyudev import UdevPoller +from octoprobe.util_subprocess import subprocess_run + +from testbed_micropython.tentacle_spec import TentacleMicropython + +from ..testcollection.constants import ( + ENV_PYTHONUNBUFFERED, + MICROPYTHON_DIRECTORY_TESTS, +) + +DIRECTORY_OF_THIS_FILE = pathlib.Path(__file__).parent + +logger = logging.getLogger(__file__) + + +class EnumTest(enum.StrEnum): + RUN_TESTS_ALL = enum.auto() + RUN_TESTS_BASIC = enum.auto() + RUN_TESTS_BASIC_B = enum.auto() + RUN_TESTS_BASIC_B_INT = enum.auto() + RUN_TESTS_BASIC_B_INT_POW = enum.auto() + RUN_TESTS_EXTMOD = enum.auto() + SERIAL_TEST = enum.auto() + SIMPLE_SERIAL_WRITE = enum.auto() + + def get_test_args(self, tentacle_test: TentacleMicropython) -> TestArgs: + serial_speed_default = 249 # kByts/s + serial_speed = serial_speed_default + for mcu, _serial_speed in ( + ("ESP32", 12), + ("LOLIN_D1", 12), + ("WB55", 12), + ("ADA_ITSYBITSY_M0", 140), + ): + if mcu in tentacle_test.description_short: + serial_speed = _serial_speed + print( + f"*** {mcu}: serial_speed_default={serial_speed_default}, serial_speed={serial_speed}kBytes/s" + ) + break + + if self is EnumTest.RUN_TESTS_ALL: + return TestArgs( + timeout_s=240.0 * 1.5, + program=["run-tests.py", "--jobs=1"], + files=[ + "--exclude=ports/rp2/rp2_lightsleep_thread.py", # Broken test + ], + ) + if self is EnumTest.RUN_TESTS_BASIC: + return TestArgs( + timeout_s=61.0 * 1.5, + program=["run-tests.py", "--jobs=1"], + files=[ + "--include=basics/*", + ], + ) + if self is EnumTest.RUN_TESTS_EXTMOD: + return TestArgs( + timeout_s=77.0 * 1.5, + program=["run-tests.py", "--jobs=1"], + files=[ + "--include=extmod/*", + ], + ) + if self is EnumTest.RUN_TESTS_BASIC_B: + return TestArgs( + program=["run-tests.py", "--jobs=1"], + timeout_s=17.0 * 1.5, + files=[ + "--include=basics/b", + ], + ) + if self is EnumTest.RUN_TESTS_BASIC_B_INT: + return TestArgs( + program=["run-tests.py", "--jobs=1"], + timeout_s=22.0 * 1.5, + files=[ + "--include=basics/(b|int_)", + ], + ) + if self is EnumTest.RUN_TESTS_BASIC_B_INT_POW: + return TestArgs( + program=["run-tests.py", "--jobs=1"], + timeout_s=13.0 * 1.5, + files=[ + "--include=basics/(b|int_)", + "--exclude=basics/builtin_pow", + ], + ) + if self is EnumTest.SERIAL_TEST: + return TestArgs( + program=["serial_test.py", "--time-per-subtest=10"], + timeout_s=90.0 * 1.5, + files=[], + ) + if self is EnumTest.SIMPLE_SERIAL_WRITE: + duration_factor = 1 + duration_factor = 5 + duration_factor = 2 + duration_factor = 4 + # duration_factor = 100 + count = int(duration_factor * 10000 * serial_speed / serial_speed_default) + count = max(1000, count) + return TestArgs( + program=[ + "simple_serial_write.py", + f"--count={count}", + ], + timeout_s=duration_factor * 3.4 * 1.5 + 10.0, + files=[], + ) + raise ValueError(self) + + +@dataclasses.dataclass(repr=True, frozen=True) +class TestArgs: + program: list[str] + timeout_s: float + files: list[str] + + +def run_test( + tentacle_test: TentacleMicropython, + repo_micropython_tests: pathlib.Path, + directory_results: pathlib.Path, + test: EnumTest, +) -> None: + assert isinstance(tentacle_test, TentacleMicropython) + assert isinstance(repo_micropython_tests, pathlib.Path) + assert isinstance(directory_results, pathlib.Path) + assert isinstance(test, EnumTest) + + print(f"*** power up tentacle_test: {tentacle_test.label_short}") + + # tentacle_test.power.dut = True + with UdevPoller() as udev: + tty = tentacle_test.dut.dut_mcu.application_mode_power_up( + tentacle=tentacle_test, + udev=udev, + ) + time.sleep(1.0) + + test_args = test.get_test_args(tentacle_test) + + args_aux: list[str] = [] + cwd = repo_micropython_tests / MICROPYTHON_DIRECTORY_TESTS + if test == EnumTest.SERIAL_TEST: + pass + elif test == EnumTest.SIMPLE_SERIAL_WRITE: + cwd = DIRECTORY_OF_THIS_FILE + else: + args_aux = [f"--result-dir={directory_results}"] + args = [ + sys.executable, + *test_args.program, + f"--test-instance=port:{tty}", + *args_aux, + *test_args.files, + # "misc/cexample_class.py", + ] + env = ENV_PYTHONUNBUFFERED + print(f"*** RUN: run_test(): subprocess_run({args})") + begin_s = time.monotonic() + subprocess_run( + args=args, + cwd=cwd, + env=env, + # logfile=testresults_directory(f"run-tests-{test_dir}.txt").filename, + logfile=directory_results + / f"testresults_{tentacle_test.tentacle_serial_number}_{tentacle_test.tentacle_spec_base.tentacle_tag}.txt", + timeout_s=test_args.timeout_s, + # TODO: Remove the following line as soon returncode of 'run-multitest.py' is fixed. + # success_returncodes=[0, 1], + ) + duration_s = time.monotonic() - begin_s + if duration_s > test_args.timeout_s * 0.5: + print( + f"*** WARNING: {tentacle_test.description_short}: duration_s={duration_s:0.0f}s > timeout*0.5={test_args.timeout_s * 0.5:0.0f}s" + ) + print(f"DONE: run_test(): subprocess_run({args})") diff --git a/src/testbed_micropython/mptest/cli.py b/src/testbed_micropython/mptest/cli.py index fdef0a8..797ebad 100644 --- a/src/testbed_micropython/mptest/cli.py +++ b/src/testbed_micropython/mptest/cli.py @@ -390,7 +390,7 @@ def report( envvar="TESTBED_MICROPYTHON_TESTRESULTS", help="Directory containing results", ), - ] = DIRECTORY_TESTRESULTS_DEFAULT, + ] = str(DIRECTORY_TESTRESULTS_DEFAULT), url: TyperAnnotated[ str, typer.Option( diff --git a/src/testbed_micropython/util_subprocess_tentacle.py b/src/testbed_micropython/util_subprocess_tentacle.py index 41866e4..57d5efa 100644 --- a/src/testbed_micropython/util_subprocess_tentacle.py +++ b/src/testbed_micropython/util_subprocess_tentacle.py @@ -76,7 +76,12 @@ def __enter__(self) -> TentaclePowerOffTimer: self._thread.start() return self - def __exit__(self, exc_type:typing.Any, value:typing.Any, traceback:typing.Any) -> None: + def __exit__( + self, + exc_type: typing.Any, + value: typing.Any, + traceback: typing.Any, + ) -> None: # Ensure timer is cancelled and thread is finished self.cancel() @@ -127,7 +132,7 @@ def tentacle_subprocess_run( try: assert logfile is not None logger.info(f"EXEC {args_text}") - logger.info(f"EXEC cwd={cwd}") + logger.info(f"EXEC cwd: {cwd}") logger.info(f"EXEC stdout: {logfile}") logfile.parent.mkdir(parents=True, exist_ok=True) with logfile.open("w") as f: @@ -228,7 +233,7 @@ def sub_run() -> subprocess.CompletedProcess[str]: def log(f: typing.Callable[[str], None]) -> None: f(f"EXEC {args_text}") - f(f" cwd={cwd}") + f(f" cwd: {cwd}") f(f" returncode: {proc.returncode}") f(f" success_codes: {success_returncodes}") f(f" duration: {time.monotonic() - begin_s:0.3f}s")