Implement mpstress

.gitignore

@@ -14,3 +14,5 @@ src/testbed_micropython/mpbuild/tests/results/*.txt
 src/testbed_micropython/mpbuild/docker-build-micropython-esp32/esp-idf
 src/testbed/experiments/*
 testresults*/**
+src/testbed_micropython/mpstress/testresults*/**
+src/testbed_micropython/mpstress/c/mpremote_c

.vscode/launch.json

@@ -135,13 +135,14 @@
                 // "--only-test=RUN-TESTS_STANDARD",
                 // "--only-test=RUN-TESTS_STANDARD_NATIVE",
                 // "--only-test=RUN-TESTS_STANDARD_VIA_MPY",
-                "--only-test=RUN-TESTS_EXTMOD_HARDWARE",
+                // "--only-test=RUN-TESTS_EXTMOD_HARDWARE",
                 // "--only-test=RUN-FLASH_FORMAT",
                 // "--only-test=RUN-TESTS_STANDARD_NATIVE",
                 // "--only-test=RUN-TESTS_STANDARD",
                 // "--skip-test=RUN-TESTS_STANDARD",
                 // "--only-test=RUN-TESTS_STANDARD:run-tests.py --via-mpy --test-dirs=micropython",
                 // "--only-test=RUN-TESTS_STANDARD:run-tests.py --test-dirs=micropython",
+                "--only-test=RUN-TESTS_STANDARD:run-tests.py --test-dirs=dummy",
                 // "--only-board=LOLIN_D1_MINI",
                 // "--skip-test=RUN-MULTITESTS_MULTINET",
                 // "--only-test=RUN-NATMODTESTS",
@@ -213,6 +214,59 @@
             "justMyCode": false,
             "subProcess": false,
         },
+        {
+            "name": "mpstress",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "testbed_micropython.mpstress.cli",
+            "cwd": "${workspaceFolder}",
+            "args": [
+                // "--help",
+                // "stress",
+                // "--micropython-tests=${workspaceFolder}/../fork_micropython",
+                "--micropython-tests=${workspaceFolder}/../micropython",
+                "--stress-scenario=NONE",
+                // "--stress-scenario=DUT_ON_OFF",
+                // "--stress-scenario=INFRA_MPREMOTE",
+                // "--stress-scenario=SUBPROCESS_INFRA_MPREMOTE",
+                // "--stress-scenario=SUBPROCESS_INFRA_MPREMOTE_C",
+                // "--test=RUN_TESTS_BASIC_B_INT_POW",
+                // "--test=SERIAL_TEST",
+                "--test=SIMPLE_SERIAL_WRITE",
+                "--stress-tentacle-count=99",
+                // "--tentacle=5f2a", // 5f2a-ADA_ITSYBITSY_M0
+                // "--tentacle=5f2c", // 5f2c-RPI_PICO_W
+                // "--tentacle=3c2a", // 3c2a-ARDUINO_NANO_33
+                // "--tentacle=0c30", // 0c30-ESP32_C3_DEVKIT
+                // "--tentacle=5d21", // 5d21-ESP32_DEVKIT
+                // "--tentacle=2d2d", // 2d2d_LOLIN_D1_MINI
+                // "--tentacle=3a21", // 3a21-PYBV11
+            ],
+            "console": "integratedTerminal",
+            "env": {
+                "PYDEVD_DISABLE_FILE_VALIDATION": "1",
+                "OCTOPROBE_ENABLE_FTRACE_MARKER": "1"
+            },
+            "justMyCode": false,
+            "subProcess": false,
+        },
+        {
+            "name": "simple_serial_write",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "testbed_micropython.mpstress.simple_serial_write",
+            "cwd": "${workspaceFolder}",
+            "args": [
+                "--count=10000",
+                "--test-instance=port:/dev/ttyUSB0"
+            ],
+            "console": "integratedTerminal",
+            "env": {
+                "PYDEVD_DISABLE_FILE_VALIDATION": "1",
+            },
+            "justMyCode": false,
+            "subProcess": false,
+        },
         {
             "name": "pytest",
             "type": "debugpy",

pyproject.toml

@@ -43,6 +43,7 @@ Repository = "https://github.com/octoprobe/octoprobe"
 
 [project.scripts]
 mptest = "testbed_micropython.mptest.cli:app"
+mpstress = "testbed_micropython.mpstress.cli:app"
 
 [project.optional-dependencies]
 

src/testbed_micropython/mpstress/README_flakyness.md

@@ -0,0 +1,136 @@
+# Testresults
+
+### --stress_scenario==NONE --test=RUN_TESTS_ALL
+
+12 tentacles
+--> no error!
+
+### --stress_scenario==DUT_ON_OFF --test=RUN_TESTS_ALL
+
+12 tentacles
+--> no error!
+
+### --stress_scenario==INFRA_MPREMOTE --test=RUN_TESTS_BASIC_B_INT_POW
+
+12 tentacles
+--> error after 20s - sometimes
+
+### --stress_scenario==INFRA_MPREMOTE --test=RUN_TESTS_BASIC_B_INT_POW --stress-tentacle-count=5
+
+5 tentacles
+--> no error!
+
+### --stress_scenario==SUBPROCESS_INFRA_MPREMOTE --test=RUN_TESTS_ALL
+
+12 tentacles
+--> no error!
+
+### --stress_scenario==SUBPROCESS_INFRA_MPREMOTE_C --test=RUN_TESTS_ALL
+
+12 tentacles
+--> no error!
+
+### --stress_scenario==INFRA_MPREMOTE --test=SERIAL_TEST
+
+12 tentacles
+--> error after 3s
+
+### --stress_scenario==INFRA_MPREMOTE --test=SERIAL_TEST --stress-tentacle-count=10
+
+10 tentacles
+--> error after 1.5s, 2s, 8s
+
+### --stress_scenario==INFRA_MPREMOTE --test=SERIAL_TEST --stress-tentacle-count=7
+
+7 tentacles
+--> error after 8s, 14s, 26s
+
+### --stress_scenario==INFRA_MPREMOTE --test=SERIAL_TEST --stress-tentacle-count=6
+
+6 tentacles
+--> no error!
+
+### --stress_scenario==INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE
+
+`mpstress --stress_scenario==INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE --tentacle=5f2c --micropython-tests=/home/octoprobe/gits/micropython`
+
+12 tentacles
+--> error after 4s
+    006000: 197kBytes/s
+    ERROR, read_duration_s=1.001166s
+        expected: b'_ABCDEFGHIJKLMNOPQRSTUVWXYabcdefghijklmnopqrstuvwxy1234567890_'
+        received: b'_ABCDEFGHIJKLMNOPQRSTUVWXYabcdefghijklmnopqrstuvwxy123456789'
+        try reading again: b'' read_duration_s=1.001263s
+
+    # Debug output from serialposix.py:
+    read(6(6)) duration=0.000011s
+    [4] = select(1.000s) duration=0.000018s
+    read(44(62)) duration=0.000010s
+    [] = select(1.000s) duration=1.000669s
+    TIMEOUT!
+    ERROR, read_duration_s=1.000970s
+    expected: b'_ABCDEFGHIJKLMNOPQRSTUVWXYabcdefghijklmnopqrstuvwxy1234567890_'
+    received: b'_ABCDEFGHIJKLMNOPQRSTUVWXYabcdefghijklmnopqr'
+    [] = select(1.000s) duration=1.000536s
+    TIMEOUT!
+    try reading again: b'' read_duration_s=1.008089s
+
+
+### --stress_scenario==INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE --stress-tentacle-count=8
+
+8 tentacles
+--> error after 9s, 19s
+
+### --stress_scenario==INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE --stress-tentacle-count=7
+
+7 tentacles
+--> error after 5s, 25s
+
+### --stress_scenario==NONE --test=SIMPLE_SERIAL_WRITE
+
+12 tentacles
+--> no error
+
+
+
+### --stress_scenario==INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE
+
+==> 5f2c connected to RHS B7
+`mpstress --stress_scenario==INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE --tentacle=5f2c --micropython-tests=/home/octoprobe/gits/micropython`
+
+12 tentacles
+--> error after 4s
+
+==> 5f2c connected to USB on computer rear
+
+`mpstress --stress_scenario==INFRA_MPREMOTE --test=SIMPLE_SERIAL_WRITE --tentacle=5f2c --micropython-tests=/home/octoprobe/gits/micropython`
+
+12 tentacles
+--> error after 120s, 35s
+--> no error 340s, 340s
+
+
+## How to run test with ftrace
+
+* tty_open: https://github.com/torvalds/linux/blob/master/drivers/tty/tty_io.c#L467
+
+```bash
+sudo chmod a+w /sys/kernel/tracing/trace_marker
+cd /tmp/ftrace
+sudo trace-cmd record -p function_graph \
+  -l tty_open \
+  -l tty_poll \
+  -l tty_release \
+  -l tty_read \
+  -l tty_write \
+  -l usb_serial_open \
+  -l acm_open
+```
+
+```bash
+mpstress --micropython-tests=/home/octoprobe/work_octoprobe/micropython --stress-scenario=NONE --test=SIMPLE_SERIAL_WRITE --stress-tentacle-count=99 2>&1 | tee > ./src/testbed_micropython/mpstress/ftrace/mpstress.log
+```
+
+```bash
+trace-cmd report
+```

src/testbed_micropython/mpstress/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.5"

src/testbed_micropython/mpstress/c/Makefile

@@ -0,0 +1,12 @@
+CC=gcc
+CFLAGS=-Wall -Wextra -std=c99
+TARGET=mpremote_c
+SOURCE=mpremote_c.c
+
+$(TARGET): $(SOURCE)
+	$(CC) $(CFLAGS) -o $(TARGET) $(SOURCE)
+
+clean:
+	rm -f $(TARGET)
+
+.PHONY: clean

src/testbed_micropython/mpstress/c/mpremote_c.c

@@ -0,0 +1,147 @@
+/*
+This programs open a serial line to a micropython cpu.
+
+Send "print('hello')"
+Return 0 if the response in "hellon\r\n"
+Return 1 on error.
+*/
+#define _DEFAULT_SOURCE
+#define _BSD_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <termios.h>
+#include <errno.h>
+
+#ifndef CRTSCTS
+#define CRTSCTS 0
+#endif
+
+int main(int argc, char *argv[]) {
+    if (argc != 2) {
+        fprintf(stderr, "Usage: %s <serial_device>\n", argv[0]);
+        fprintf(stderr, "Example: %s /dev/ttyACM4\n", argv[0]);
+        return 1;
+    }
+
+    const char *serial_device = argv[1];
+    int serial_fd;
+    struct termios tty;
+    const char *command = "print('hello')\r\n";
+    char response[256];
+    const char *expected_response = "hello\r\n";
+    ssize_t bytes_read;
+
+    // Open serial device
+    serial_fd = open(serial_device, O_RDWR | O_NOCTTY);
+    if (serial_fd < 0) {
+        perror("Error opening serial device");
+        return 1;
+    }
+
+    // Get current terminal attributes
+    if (tcgetattr(serial_fd, &tty) != 0) {
+        perror("Error getting terminal attributes");
+        close(serial_fd);
+        return 1;
+    }
+
+    // Configure serial port settings
+    // Set baud rate to 115200
+    cfsetospeed(&tty, B115200);
+    cfsetispeed(&tty, B115200);
+
+    // 8N1 mode
+    tty.c_cflag &= ~PARENB;        // No parity
+    tty.c_cflag &= ~CSTOPB;        // One stop bit
+    tty.c_cflag &= ~CSIZE;         // Clear size bits
+    tty.c_cflag |= CS8;            // 8 data bits
+    tty.c_cflag &= ~CRTSCTS;       // No hardware flow control
+    tty.c_cflag |= CREAD | CLOCAL; // Enable reading, ignore modem control lines
+
+    // Input modes
+    tty.c_iflag &= ~(IXON | IXOFF | IXANY); // No software flow control
+    tty.c_iflag &= ~(ICANON | ECHO | ECHOE | ISIG); // Raw input
+
+    // Output modes
+    tty.c_oflag &= ~OPOST; // Raw output
+
+    // Local modes
+    tty.c_lflag &= ~(ICANON | ECHO | ECHOE | ISIG); // Raw mode
+
+    // Control characters
+    tty.c_cc[VMIN] = 2*strlen(expected_response);   // Blocking read - wait for at least 1 character
+    tty.c_cc[VTIME] = 10; // 1 second timeout (in deciseconds)
+
+    // Apply settings
+    if (tcsetattr(serial_fd, TCSANOW, &tty) != 0) {
+        perror("Error setting terminal attributes");
+        close(serial_fd);
+        return 1;
+    }
+
+    // Flush any existing data
+    tcflush(serial_fd, TCIOFLUSH);
+
+    // Give device time to initialize
+    // usleep(100000); // 100ms
+
+    // Send command
+    ssize_t bytes_written = write(serial_fd, command, strlen(command));
+    if (bytes_written < 0) {
+        perror("Error writing to serial device");
+        close(serial_fd);
+        return 1;
+    }
+
+    // Give device time to process and respond
+    // usleep(500000); // 500ms
+
+    // Read response
+    memset(response, 0, sizeof(response));
+    bytes_read = read(serial_fd, response, sizeof(response) - 1);
+    if (bytes_read < 0) {
+        perror("Error reading from serial device");
+        close(serial_fd);
+        return 1;
+    }
+
+    // Close serial device
+    close(serial_fd);
+
+    // Check if we got the expected response
+    // Look for "hello" followed by newline and carriage return
+    if (strstr(response, "hello") != NULL) {
+        // Check if response contains hello followed by \r\n or \n\r
+        if (strstr(response, "hello\r\n") != NULL || strstr(response, "hello\n\r") != NULL) {
+            printf("Success: Got expected response containing 'hello\\r\\n' or 'hello\\n\\r'\n");
+            return 0;
+        } else if (strstr(response, "hello") != NULL) {
+            printf("Partial success: Got 'hello' but not with expected line endings\n");
+            printf("Response was: ");
+            for (int i = 0; i < bytes_read; i++) {
+                if (response[i] >= 32 && response[i] <= 126) {
+                    printf("%c", response[i]);
+                } else {
+                    printf("\\x%02x", (unsigned char)response[i]);
+                }
+            }
+            printf("\n");
+            return 0; // Still consider it success if we got hello
+        }
+    }
+
+    printf("Failed: Did not get expected response 'hello\\n\\r'\n");
+    printf("Response was: ");
+    for (int i = 0; i < bytes_read; i++) {
+        if (response[i] >= 32 && response[i] <= 126) {
+            printf("%c", response[i]);
+        } else {
+            printf("\\x%02x", (unsigned char)response[i]);
+        }
+    }
+    printf("\n");
+    return 1;
+}