From 2bbcd7c04732b7d012edbbb19e44127abbe8ce26 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Fri, 19 Dec 2025 13:06:59 +0100 Subject: [PATCH 01/11] feat: simulate link failure --- netsim/main.py | 34 ++++++++++++++++ netsim/sims/integration/adverse.json | 60 ++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/netsim/main.py b/netsim/main.py index cddd9c1..b0a14c0 100644 --- a/netsim/main.py +++ b/netsim/main.py @@ -5,6 +5,7 @@ import os import sys import tempfile +import threading import time from mininet.log import setLogLevel, info, error @@ -271,6 +272,37 @@ def prep_net(net, prefix, sniff): return sniffer +def schedule_mid_run_actions(net, nodes, node_counts, node_ips, runner_id, prefix): + """Schedule mid-run actions like blocking direct connections.""" + for node in nodes: + if "mid_run_action" not in node: + continue + action = node["mid_run_action"] + for i in range(int(node["count"])): + node_name = f'{node["name"]}_{i}_r{runner_id}' + if "block_direct_to" in action: + target = action["block_direct_to"] + target_cnt = node_counts.get(target, 1) + target_name = f'{target}_{i % target_cnt}_r{runner_id}' + target_ip = node_ips.get(target_name) + if not target_ip: + continue + delay = action.get("delay_seconds", 5) + n = net.get(node_name) + + def block_after_delay(node_ref, ip, delay_sec, log_prefix, src_name): + time.sleep(delay_sec) + node_ref.cmd(f"iptables -A OUTPUT -d {ip} -p udp -j DROP") + info(f"[{log_prefix}] Blocked direct UDP to {ip} from {src_name}\n") + + t = threading.Thread( + target=block_after_delay, + args=(n, target_ip, delay, prefix, node_name) + ) + t.daemon = True + t.start() + + def run_case(nodes, runner_id, prefix, args, debug=False, visualize=False): topo = StarTopo(nodes=nodes, runner_id=runner_id) net = Mininet(topo=topo, waitConnected=True, link=TCLink) @@ -314,6 +346,8 @@ def run_case(nodes, runner_id, prefix, args, debug=False, visualize=False): # CLI(net) + schedule_mid_run_actions(net, nodes, node_counts, node_ips, runner_id, prefix) + process_errors = monitor_short_processes(p_short_box, prefix) if process_errors: error("\n" + "=" * 80 + "\n") diff --git a/netsim/sims/integration/adverse.json b/netsim/sims/integration/adverse.json index c22e9f0..9c4b1fc 100644 --- a/netsim/sims/integration/adverse.json +++ b/netsim/sims/integration/adverse.json @@ -276,6 +276,66 @@ } } ] + }, + { + "name": "direct_to_relay_fallback", + "description": "Direct connection blocked mid-transfer, falls back to relay", + "visualize": true, + "nodes": [ + { + "name": "1_r", + "count": 1, + "cmd": "./bins/iroh-relay --dev --config-path ./data/relay.cfg", + "type": "public", + "wait": 2, + "connect": { + "strategy": "none" + } + }, + { + "name": "2_d", + "count": 1, + "cmd": "./bins/iroh-dns-server --config ./data/dns.test.cfg", + "type": "public", + "wait": 2, + "connect": { + "strategy": "none" + } + }, + { + "name": "i_srv", + "count": 1, + "cmd": "./bins/iroh-transfer provide --env dev --size=50M --relay-url=\"http://10.0.0.1:3340\" --pkarr-relay-url=\"http://10.0.0.2:8080/pkarr\" --dns-origin-domain=\"10.0.0.2:5300\"", + "type": "public", + "wait": 10, + "connect": { + "strategy": "none" + }, + "param_parser": "iroh_endpoint_with_addrs" + }, + { + "name": "i_get", + "count": 1, + "cmd": "time ./bins/iroh-transfer fetch --env dev --relay-url=\"http://10.0.0.1:3340\" --remote-relay-url=\"http://10.0.0.1:3340\" --pkarr-relay-url=\"http://10.0.0.2:8080/pkarr\" --dns-origin-domain=\"10.0.0.2:5300\" --remote-direct-address=\"%s\" %s", + "type": "public", + "connect": { + "strategy": "params_with_parsed_addrs", + "node": "i_srv" + }, + "process": "short", + "parser": "iroh_cust_50M", + "integration": "magic_iroh_client", + "link": { + "bw": 8, + "latency": 50, + "loss": 0 + }, + "mid_run_action": { + "delay_seconds": 6, + "block_direct_to": "i_srv" + } + } + ] } ] } From fefc32ae0b426ebd592889c0b9ccf2250728c1ee Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Fri, 19 Dec 2025 13:16:07 +0100 Subject: [PATCH 02/11] persist logs for a bit --- .github/workflows/netsim_integration.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/netsim_integration.yml b/.github/workflows/netsim_integration.yml index bdf6eeb..86cb8e4 100644 --- a/.github/workflows/netsim_integration.yml +++ b/.github/workflows/netsim_integration.yml @@ -62,6 +62,14 @@ jobs: sudo mn --clean sudo python3 main.py --integration --max-workers=4 sims/integration + - name: Upload logs + uses: actions/upload-artifact@v4 + if: always() + with: + name: netsim-logs + path: netsim/logs/ + retention-days: 7 + - name: Setup Environment (PR) if: ${{ github.event_name == 'pull_request' }} shell: bash From cd047088e25fc828699fedfba19d17a7b1e3bee3 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Fri, 19 Dec 2025 13:26:03 +0100 Subject: [PATCH 03/11] fix --- netsim/sims/integration/adverse.json | 96 ++++++++++++++-------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/netsim/sims/integration/adverse.json b/netsim/sims/integration/adverse.json index 9c4b1fc..798892c 100644 --- a/netsim/sims/integration/adverse.json +++ b/netsim/sims/integration/adverse.json @@ -15,7 +15,12 @@ "connect": { "strategy": "none" }, - "param_parser": "iroh_endpoint_with_addrs" + "param_parser": "iroh_endpoint_with_addrs", + "link": { + "bw": 4, + "latency": 200, + "loss": 0 + } }, { "name": "i_get", @@ -28,12 +33,7 @@ }, "process": "short", "parser": "iroh_cust_10M", - "integration": "magic_iroh_client", - "link": { - "bw": 4, - "latency": 200, - "loss": 0 - } + "integration": "magic_iroh_client" } ] }, @@ -51,7 +51,12 @@ "connect": { "strategy": "none" }, - "param_parser": "iroh_endpoint_with_addrs" + "param_parser": "iroh_endpoint_with_addrs", + "link": { + "bw": 8, + "latency": 200, + "loss": 1 + } }, { "name": "i_get", @@ -64,12 +69,7 @@ }, "process": "short", "parser": "iroh_cust_10M", - "integration": "magic_iroh_client", - "link": { - "loss": 1, - "bw": 8, - "latency": 200 - } + "integration": "magic_iroh_client" } ] }, @@ -107,7 +107,12 @@ "connect": { "strategy": "none" }, - "param_parser": "iroh_endpoint_with_addrs" + "param_parser": "iroh_endpoint_with_addrs", + "link": { + "bw": 4, + "latency": 200, + "loss": 0 + } }, { "name": "i_get", @@ -120,12 +125,7 @@ }, "process": "short", "parser": "iroh_cust_10M", - "integration": "magic_iroh_client", - "link": { - "bw": 4, - "latency": 200, - "loss": 0 - } + "integration": "magic_iroh_client" } ] }, @@ -163,7 +163,12 @@ "connect": { "strategy": "none" }, - "param_parser": "iroh_endpoint_with_addrs" + "param_parser": "iroh_endpoint_with_addrs", + "link": { + "bw": 8, + "latency": 200, + "loss": 1 + } }, { "name": "i_get", @@ -176,12 +181,7 @@ }, "process": "short", "parser": "iroh_cust_10M", - "integration": "magic_iroh_client", - "link": { - "loss": 1, - "bw": 8, - "latency": 200 - } + "integration": "magic_iroh_client" } ] }, @@ -209,7 +209,12 @@ "connect": { "strategy": "none" }, - "param_parser": "iroh_endpoint_with_addrs" + "param_parser": "iroh_endpoint_with_addrs", + "link": { + "bw": 4, + "latency": 200, + "loss": 0 + } }, { "name": "i_get", @@ -222,12 +227,7 @@ }, "process": "short", "parser": "iroh_cust_10M", - "integration": "magic_iroh_client", - "link": { - "bw": 4, - "latency": 200, - "loss": 0 - } + "integration": "magic_iroh_client" } ] }, @@ -255,7 +255,12 @@ "connect": { "strategy": "none" }, - "param_parser": "iroh_endpoint_with_addrs" + "param_parser": "iroh_endpoint_with_addrs", + "link": { + "bw": 8, + "latency": 200, + "loss": 1 + } }, { "name": "i_get", @@ -268,12 +273,7 @@ }, "process": "short", "parser": "iroh_cust_10M", - "integration": "magic_iroh_client", - "link": { - "loss": 1, - "bw": 8, - "latency": 200 - } + "integration": "magic_iroh_client" } ] }, @@ -311,7 +311,12 @@ "connect": { "strategy": "none" }, - "param_parser": "iroh_endpoint_with_addrs" + "param_parser": "iroh_endpoint_with_addrs", + "link": { + "bw": 8, + "latency": 50, + "loss": 0 + } }, { "name": "i_get", @@ -325,11 +330,6 @@ "process": "short", "parser": "iroh_cust_50M", "integration": "magic_iroh_client", - "link": { - "bw": 8, - "latency": 50, - "loss": 0 - }, "mid_run_action": { "delay_seconds": 6, "block_direct_to": "i_srv" From 680249b364abc9cc4d63932dc1dd7db75ea68ddf Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Fri, 19 Dec 2025 13:35:56 +0100 Subject: [PATCH 04/11] testing - needs reverting --- .github/workflows/netsim_integration.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/netsim_integration.yml b/.github/workflows/netsim_integration.yml index 86cb8e4..a8fe31e 100644 --- a/.github/workflows/netsim_integration.yml +++ b/.github/workflows/netsim_integration.yml @@ -60,7 +60,8 @@ jobs: cd netsim sudo kill -9 $(pgrep ovs) || true sudo mn --clean - sudo python3 main.py --integration --max-workers=4 sims/integration + # TODO: revert to full suite: sudo python3 main.py --integration --max-workers=4 sims/integration + sudo python3 main.py --integration sims/integration/adverse.json --skip adverse_iroh__direct_throttled,adverse_iroh__direct_lossy,adverse_iroh__relay_dns_throttled,adverse_iroh__relay_dns_lossy,adverse_iroh__nat_both_throttled,adverse_iroh__nat_both_lossy - name: Upload logs uses: actions/upload-artifact@v4 From 45bdbe12279fcb9b9121b5bae2f7d03265c38f94 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Fri, 19 Dec 2025 13:41:34 +0100 Subject: [PATCH 05/11] improve logging --- netsim/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/netsim/main.py b/netsim/main.py index b0a14c0..d25d03d 100644 --- a/netsim/main.py +++ b/netsim/main.py @@ -387,7 +387,7 @@ def run(case, runner_id, name, args): viz = False if "visualize" in case: viz = case["visualize"] & args.visualize - print('Running "%s"...' % prefix) + print('Running "%s"...' % prefix, flush=True) n, s = (None, None) if not args.reports_only: (n, s) = run_case(nodes, runner_id, prefix, args, args.debug, viz) @@ -410,7 +410,7 @@ def run_parallel(cases, name, skiplist, args, max_workers=4): for case in cases: prefix = name + "__" + case["name"] if prefix in skiplist: - print("Skipping:", prefix) + print("Skipping:", prefix, flush=True) else: filtered.append(case) @@ -486,14 +486,14 @@ def run_parallel(cases, name, skiplist, args, max_workers=4): else: paths.append(args.cfg) - print("Args:", args) + print("Args:", args, flush=True) for path in paths: config_f = open(path, "r") config = json.load(config_f) config_f.close() name = config["name"] - print(f"Start testing: %s\n" % path) + print(f"Start testing: %s" % path, flush=True) run_parallel(config["cases"], name, skiplist, args, args.max_workers) write_failure_summary() From a804c53d4f2929319605bd652c9a5e0f5a2ddc9f Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Fri, 19 Dec 2025 13:42:19 +0100 Subject: [PATCH 06/11] testing - needs reverting --- .github/workflows/netsim_integration.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/netsim_integration.yml b/.github/workflows/netsim_integration.yml index a8fe31e..d48d58e 100644 --- a/.github/workflows/netsim_integration.yml +++ b/.github/workflows/netsim_integration.yml @@ -43,7 +43,8 @@ jobs: - name: Fetch and build iroh run: | - git clone --depth 1 https://github.com/n0-computer/iroh.git + # TODO: revert to main: git clone --depth 1 https://github.com/n0-computer/iroh.git + git clone --depth 1 --branch fix-timeouts https://github.com/n0-computer/iroh.git cd iroh cargo build --release --all-features -p iroh-relay -p iroh-dns-server cargo build --release --all-features -p iroh --examples From 79db22f0492d184000d23b9058bdf8c6972728ef Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Fri, 19 Dec 2025 13:45:25 +0100 Subject: [PATCH 07/11] revert and cleanup --- .github/workflows/netsim_integration.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/netsim_integration.yml b/.github/workflows/netsim_integration.yml index d48d58e..55cc157 100644 --- a/.github/workflows/netsim_integration.yml +++ b/.github/workflows/netsim_integration.yml @@ -37,21 +37,15 @@ jobs: sudo apt update ./setup.sh - - name: Build chuck - run: | - cargo build --release - - name: Fetch and build iroh run: | - # TODO: revert to main: git clone --depth 1 https://github.com/n0-computer/iroh.git - git clone --depth 1 --branch fix-timeouts https://github.com/n0-computer/iroh.git + git clone --depth 1 https://github.com/n0-computer/iroh.git cd iroh cargo build --release --all-features -p iroh-relay -p iroh-dns-server cargo build --release --all-features -p iroh --examples - name: Copy binaries to right location run: | - cp target/release/chuck netsim/bins/chuck cp iroh/target/release/iroh-relay netsim/bins/iroh-relay cp iroh/target/release/iroh-dns-server netsim/bins/iroh-dns-server cp iroh/target/release/examples/transfer netsim/bins/iroh-transfer @@ -61,8 +55,7 @@ jobs: cd netsim sudo kill -9 $(pgrep ovs) || true sudo mn --clean - # TODO: revert to full suite: sudo python3 main.py --integration --max-workers=4 sims/integration - sudo python3 main.py --integration sims/integration/adverse.json --skip adverse_iroh__direct_throttled,adverse_iroh__direct_lossy,adverse_iroh__relay_dns_throttled,adverse_iroh__relay_dns_lossy,adverse_iroh__nat_both_throttled,adverse_iroh__nat_both_lossy + sudo python3 main.py --integration --max-workers=4 sims/integration - name: Upload logs uses: actions/upload-artifact@v4 From 3fb728df487d123efb3c9400d6d9c4feb52746ed Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Fri, 19 Dec 2025 13:52:32 +0100 Subject: [PATCH 08/11] testing -- needs reverting --- .github/workflows/netsim_integration.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/netsim_integration.yml b/.github/workflows/netsim_integration.yml index 55cc157..4a3ea36 100644 --- a/.github/workflows/netsim_integration.yml +++ b/.github/workflows/netsim_integration.yml @@ -39,7 +39,8 @@ jobs: - name: Fetch and build iroh run: | - git clone --depth 1 https://github.com/n0-computer/iroh.git + # TODO: revert to main: git clone --depth 1 https://github.com/n0-computer/iroh.git + git clone --depth 1 --branch fix-use-per-path-config https://github.com/n0-computer/iroh.git cd iroh cargo build --release --all-features -p iroh-relay -p iroh-dns-server cargo build --release --all-features -p iroh --examples @@ -55,7 +56,8 @@ jobs: cd netsim sudo kill -9 $(pgrep ovs) || true sudo mn --clean - sudo python3 main.py --integration --max-workers=4 sims/integration + # TODO: revert to full suite: sudo python3 main.py --integration --max-workers=4 sims/integration + sudo python3 main.py --integration sims/integration/adverse.json --skip adverse_iroh__direct_throttled,adverse_iroh__direct_lossy,adverse_iroh__relay_dns_throttled,adverse_iroh__relay_dns_lossy,adverse_iroh__nat_both_throttled,adverse_iroh__nat_both_lossy - name: Upload logs uses: actions/upload-artifact@v4 From 0fd8b7352f154da865c3ded0bb64682d8bb244e6 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Fri, 19 Dec 2025 14:38:39 +0100 Subject: [PATCH 09/11] testing -- needs reverting --- .github/workflows/netsim_integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/netsim_integration.yml b/.github/workflows/netsim_integration.yml index 4a3ea36..561600b 100644 --- a/.github/workflows/netsim_integration.yml +++ b/.github/workflows/netsim_integration.yml @@ -40,7 +40,7 @@ jobs: - name: Fetch and build iroh run: | # TODO: revert to main: git clone --depth 1 https://github.com/n0-computer/iroh.git - git clone --depth 1 --branch fix-use-per-path-config https://github.com/n0-computer/iroh.git + git clone --depth 1 --branch update-quinn https://github.com/n0-computer/iroh.git cd iroh cargo build --release --all-features -p iroh-relay -p iroh-dns-server cargo build --release --all-features -p iroh --examples From c8bebbba2d995aadbfd95fa5bf93ca5ee1932fc3 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Fri, 19 Dec 2025 14:53:47 +0100 Subject: [PATCH 10/11] cleanup --- .github/workflows/netsim_integration.yml | 7 +++---- netsim/sims/integration/adverse.json | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/netsim_integration.yml b/.github/workflows/netsim_integration.yml index 561600b..81a140d 100644 --- a/.github/workflows/netsim_integration.yml +++ b/.github/workflows/netsim_integration.yml @@ -39,8 +39,8 @@ jobs: - name: Fetch and build iroh run: | - # TODO: revert to main: git clone --depth 1 https://github.com/n0-computer/iroh.git - git clone --depth 1 --branch update-quinn https://github.com/n0-computer/iroh.git + git clone --depth 1 https://github.com/n0-computer/iroh.git + # git clone --depth 1 --branch update-quinn https://github.com/n0-computer/iroh.git cd iroh cargo build --release --all-features -p iroh-relay -p iroh-dns-server cargo build --release --all-features -p iroh --examples @@ -56,8 +56,7 @@ jobs: cd netsim sudo kill -9 $(pgrep ovs) || true sudo mn --clean - # TODO: revert to full suite: sudo python3 main.py --integration --max-workers=4 sims/integration - sudo python3 main.py --integration sims/integration/adverse.json --skip adverse_iroh__direct_throttled,adverse_iroh__direct_lossy,adverse_iroh__relay_dns_throttled,adverse_iroh__relay_dns_lossy,adverse_iroh__nat_both_throttled,adverse_iroh__nat_both_lossy + sudo python3 main.py --integration --max-workers=4 sims/integration - name: Upload logs uses: actions/upload-artifact@v4 diff --git a/netsim/sims/integration/adverse.json b/netsim/sims/integration/adverse.json index 798892c..c366053 100644 --- a/netsim/sims/integration/adverse.json +++ b/netsim/sims/integration/adverse.json @@ -305,7 +305,7 @@ { "name": "i_srv", "count": 1, - "cmd": "./bins/iroh-transfer provide --env dev --size=50M --relay-url=\"http://10.0.0.1:3340\" --pkarr-relay-url=\"http://10.0.0.2:8080/pkarr\" --dns-origin-domain=\"10.0.0.2:5300\"", + "cmd": "./bins/iroh-transfer provide --env dev --size=25M --relay-url=\"http://10.0.0.1:3340\" --pkarr-relay-url=\"http://10.0.0.2:8080/pkarr\" --dns-origin-domain=\"10.0.0.2:5300\"", "type": "public", "wait": 10, "connect": { @@ -328,7 +328,7 @@ "node": "i_srv" }, "process": "short", - "parser": "iroh_cust_50M", + "parser": "iroh_cust_25M", "integration": "magic_iroh_client", "mid_run_action": { "delay_seconds": 6, From 03fb89ed79cec63f79bf8bcefcc5031f02f8939d Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Fri, 19 Dec 2025 15:31:29 +0100 Subject: [PATCH 11/11] testing -- needs reverting --- .github/workflows/netsim_integration.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/netsim_integration.yml b/.github/workflows/netsim_integration.yml index 81a140d..1de78f3 100644 --- a/.github/workflows/netsim_integration.yml +++ b/.github/workflows/netsim_integration.yml @@ -39,8 +39,8 @@ jobs: - name: Fetch and build iroh run: | - git clone --depth 1 https://github.com/n0-computer/iroh.git - # git clone --depth 1 --branch update-quinn https://github.com/n0-computer/iroh.git + # git clone --depth 1 https://github.com/n0-computer/iroh.git + git clone --depth 1 --branch feat-improve-network-change https://github.com/n0-computer/iroh.git cd iroh cargo build --release --all-features -p iroh-relay -p iroh-dns-server cargo build --release --all-features -p iroh --examples