Skip to content

Multi-node PR #61

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
pooriaPoorsarvi wants to merge 22 commits into
base: main
Choose a base branch
from
Draft

Multi-node PR #61

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
ca3f9a0
mutli-node internal init
pooriaPoorsarvi Nov 5, 2025
c1acfa1
set starting version for multi-node
pooriaPoorsarvi Nov 5, 2025
005d7b1
fixed dockerfile for multi-node
pooriaPoorsarvi Nov 5, 2025
cdb604e
updated host env
pooriaPoorsarvi Nov 10, 2025
1553188
added node_number support init
pooriaPoorsarvi Nov 10, 2025
2c26150
Added initial scripts to be moved later fully to qflex
pooriaPoorsarvi Nov 10, 2025
638f8ee
bumping version for node_id and new deps
pooriaPoorsarvi Nov 10, 2025
9069f8d
Added the expect files
pooriaPoorsarvi Nov 10, 2025
2b901e9
added debug mode with qemu and parallel-qemu in image
pooriaPoorsarvi Nov 11, 2025
1ca255f
fixed docker images with debugging
pooriaPoorsarvi Nov 11, 2025
1fac9ba
added host time timing to exp
pooriaPoorsarvi Nov 11, 2025
e57aa30
added quantum time with small quantum
pooriaPoorsarvi Nov 11, 2025
d2194f1
working state of client and server
pooriaPoorsarvi Nov 12, 2025
d8bf225
got existing quantum working with client and server
pooriaPoorsarvi Nov 12, 2025
c1a701a
init pdes netdev support for qemu
pooriaPoorsarvi Nov 14, 2025
6e078c1
updated how makefile builds qemu repos
pooriaPoorsarvi Nov 23, 2025
9da8f8b
temp solution to communicate
pooriaPoorsarvi Nov 29, 2025
d63108a
icount also working
pooriaPoorsarvi Nov 29, 2025
f227b13
first draft of translated time
pooriaPoorsarvi Nov 29, 2025
18d27bc
latest commit before seriliazation
pooriaPoorsarvi Feb 2, 2026
07d2e0d
temp changes for web-search multi-node
pooriaPoorsarvi Feb 3, 2026
9162c9e
checkpoint test
pooriaPoorsarvi Feb 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .bumpversion.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "3.2.1"
current_version = "3.4.0"
parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)"
serialize = ["{major}.{minor}.{patch}"]
search = "{current_version}"
Expand Down
42 changes: 9 additions & 33 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test -y

# Need to update after installing the previous dependencies
RUN apt-get update -y
RUN apt install -y --no-install-recommends net-tools
RUN apt install --yes cloud-image-utils qemu-system-x86
RUN apt install -y --no-install-recommends net-tools
RUN apt install -y --no-install-recommends wget
RUN apt install -y --no-install-recommends curl
RUN apt install -y --no-install-recommends git
Expand All @@ -42,6 +45,11 @@ RUN apt install -y --no-install-recommends zstd
RUN apt install -y --no-install-recommends vim
RUN apt install -y --no-install-recommends tmux
RUN apt install -y --no-install-recommends htop
RUN apt-get install -y expect telnet

RUN apt-get update && apt-get install -y \
iproute2 \
&& rm -rf /var/lib/apt/lists/*

# --break-system-package for ubuntu 24.04
RUN pip install conan && pip cache purge
Expand All @@ -62,38 +70,6 @@ ARG MODE=release
WORKDIR /home/dev/qflex


# TODO add a check later to make sure qflex folder it self is never mounted, as we need the binaries, or change where they are craeted
# TODO address the two qemu versions
RUN --mount=type=bind,source=./qemu,target=/home/dev/qflex/qemu,rw conan profile detect --force && \
conan build flexus -pr flexus/target/_profile/${MODE} --name=knottykraken -of /home/dev/qflex/out -b missing && \
conan build flexus -pr flexus/target/_profile/${MODE} --name=semikraken -of /home/dev/qflex/out -b missing && \
conan export-pkg flexus -pr flexus/target/_profile/${MODE} --name=knottykraken -of /home/dev/qflex/out && \
conan export-pkg flexus -pr flexus/target/_profile/${MODE} --name=semikraken -of /home/dev/qflex/out && \
conan cache clean -v && \
conan remove -c "*" && \
./build cq ${MODE} && \
python3 build-multiple-kraken_vanilla.py \
mkdir /home/dev/qflex/kraken_out && \
cp -r out/lib/Release /home/dev/qflex/kraken_out && \
rm -rf out && \
mkdir qemu-saved && \
cp -r /home/dev/qflex/qemu/pc-bios /home/dev/qflex/qemu-saved/pc-bios && \
cp -r /home/dev/qflex/qemu/build /home/dev/qflex/qemu-saved/build

RUN --mount=type=bind,source=./parallel-qemu,target=/home/dev/qflex/parallel-qemu,rw cd parallel-qemu && \
./configure --target-list=aarch64-softmmu --disable-gtk --enable-capstone && \
ninja -C build && \
mkdir /home/dev/qflex/parallel-qemu-saved && \
cp -r /home/dev/qflex/parallel-qemu/pc-bios /home/dev/qflex/parallel-qemu-saved/pc-bios && \
cp -r /home/dev/qflex/parallel-qemu/build /home/dev/qflex/parallel-qemu-saved/build




WORKDIR /home/dev/qflex
# Post-build file link
RUN ln -s /home/dev/qflex/parallel-qemu-saved/build/aarch64-softmmu/qemu-system-aarch64 /home/dev/qflex/qemu-aarch64
RUN ln -s /home/dev/qflex/parallel-qemu-saved/build/qemu-img /home/dev/qflex/qemu-img

RUN pip install -r requirements.txt
COPY ./commands /home/dev/qflex/commands
Expand All @@ -106,4 +82,4 @@ COPY ./QEMU_EFI.fd /home/dev/qflex/QEMU_EFI.fd
# TODO this is hardcoded as typer doesn't have a way to generate completions from within docker build, as long as tool is called qflex this is ok
RUN cat /home/dev/qflex/completion_docker.txt >> /root/.bashrc

CMD ["bash"]
CMD ["bash"]
23 changes: 23 additions & 0 deletions Dockerfile.qemu.debug
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# syntax=docker/dockerfile:1.17-labs

ARG BASE_IMAGE

FROM ${BASE_IMAGE}


WORKDIR /home/dev/qflex


# TODO add a check later to make sure qflex folder it self is never mounted, as we need the binaries, or change where they are craeted
# TODO address the two qemu versions
COPY ./qemu /home/dev/qflex/qemu
COPY ./parallel-qemu /home/dev/qflex/parallel-qemu


RUN make MODE=debug qemu-config && make MODE=debug qemu-build

RUN make MODE=debug parallel-qemu-config && make MODE=debug parallel-qemu-build



CMD ["bash"]
18 changes: 18 additions & 0 deletions Dockerfile.qemu.release
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# syntax=docker/dockerfile:1.17-labs

ARG BASE_IMAGE

FROM ${BASE_IMAGE}


WORKDIR /home/dev/qflex


# TODO add a check later to make sure qflex folder it self is never mounted, as we need the binaries, or change where they are craeted
# TODO address the two qemu versions
RUN --mount=type=bind,source=./qemu,target=/home/dev/qflex/qemu,rw make MODE=release qemu-config && make MODE=release qemu-build
RUN --mount=type=bind,source=./parallel-qemu,target=/home/dev/qflex/parallel-qemu,rw make MODE=release parallel-qemu-config && make MODE=release parallel-qemu-build



CMD ["bash"]
64 changes: 64 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,3 +12,67 @@ install-dev-requirements:
pip install -r requirements.docs.txt && \
pip install uv && \
uv tool install bump-my-version

qemu-config:
ifndef MODE
$(error MODE is not set. Usage: make qemu-build MODE=debug|release)
endif
conan profile detect --force && \
conan build flexus -pr flexus/target/_profile/${MODE} --name=knottykraken -of ./out -b missing && \
conan build flexus -pr flexus/target/_profile/${MODE} --name=semikraken -of ./out -b missing && \
conan export-pkg flexus -pr flexus/target/_profile/${MODE} --name=knottykraken -of ./out && \
conan export-pkg flexus -pr flexus/target/_profile/${MODE} --name=semikraken -of ./out && \
conan cache clean -v && \
conan remove -c "*" && \
cd qemu && \
./configure --target-list=aarch64-softmmu \
--disable-docs \
--enable-capstone \
--enable-slirp \
--enable-libqflex \
--enable-snapvm-external \
--disable-gtk \
$(if $(filter debug,$(MODE)),--enable-debug) && \
cd ..


# TODO this still has some config in it, move it to speed up building

qemu-move-files:
rm -rf ./qemu-saved && \
mkdir -p qemu-saved && \
cp -r ./qemu/pc-bios ./qemu-saved/pc-bios && \
cp -r ./qemu/build ./qemu-saved/build

build-kraken:
python3 build-multiple-kraken_vanilla.py && \
rm -rf ./kraken_out&& \
mkdir -p ./kraken_out && \
cp -r out/lib/Release/* ./kraken_out && \
rm -rf out


qemu-ninja:
cd qemu && \
ninja -C build && \
cd .. && \
make qemu-move-files

# TODO check if it can be replaced with qemu-ninja
qemu-build:
make -C qemu -j && \
make qemu-move-files

parallel-qemu-config:
cd parallel-qemu && \
./configure --target-list=aarch64-softmmu --disable-gtk --enable-capstone && \
cd ..

parallel-qemu-build:
cd parallel-qemu && \
ninja -C build && \
cd .. && \
rm -rf parallel-qemu-saved && mkdir -p parallel-qemu-saved && \
cp -r parallel-qemu/build parallel-qemu-saved/build && \
cp -r parallel-qemu/pc-bios parallel-qemu-saved/pc-bios && \
mv parallel-qemu-saved/build/pc-bios parallel-qemu-saved/pc-bios
2 changes: 1 addition & 1 deletion VERSION
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
3.2.1
3.4.0
12 changes: 12 additions & 0 deletions clean_up.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/bash
# Get unique PIDs from listening ports and kill them
netstat -tulpn 2>/dev/null | awk -F'/' '/LISTEN|udp/ {print $1}' | awk '{print $NF}' | grep -E '^[0-9]+$' | sort -u | while read pid; do
kill -9 "$pid" && echo "Killed $pid" || echo "Failed to kill $pid"
done

# Kill all qemu-system-aar processes
ps -a | grep 'qemu-system-aar' | grep -v grep | awk '{print $1}' | while read pid; do
kill -9 "$pid" && echo "Killed qemu process $pid" || echo "Failed to kill $pid"
done

rm /dev/shm/pdes*
25 changes: 21 additions & 4 deletions commands/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ class ExperimentContext(BaseModel):
loadvm_name: str = Field(default="", description="Name of the loadvm to use in QEMU, optional")
image_address: str = Field(default="", description="Full address of the image to use. Set up during initialization based on other parameters.")
include_affinity: bool = Field(default=False, description="Whether or not generate affinity index in core_info.csv.")
node_number: int = Field(default=-1, description="Node number in multi-node setup, -1 means single node.")

def get_mounting_folder(self) -> str:
return self.mounting_folder
Expand Down Expand Up @@ -166,6 +167,20 @@ def set_up_image(self):
# Create a symlink to the new image in the experiment folder
os.symlink(f"{self.image_folder}/experiments/{self.experiment_name}/{self.image_name}", self.get_local_image_address())
print(f"Linked image to")

if self.node_number >=0:
print(f"Setting up node-specific image for node {self.node_number}...")
old_address = self.image_address

image_name_parts = self.image_name.split('.')
new_image_name = '.'.join(image_name_parts[:-1]) + f'-node{self.node_number}.' + image_name_parts[-1]
self.image_address = self.image_address.replace(self.image_name, new_image_name)
self.image_name = new_image_name

if not os.path.exists(self.image_address):
print(f"Creating node-specific image for node {self.node_number}...")
os.system(f"cp -u {old_address} {self.image_address}")
print(f"Node-specific image address: {self.image_address}")



Expand Down Expand Up @@ -216,6 +231,7 @@ def set_up_folders(self):
"./QEMU_EFI.fd",
"./qemu-saved/build/qemu-system-aarch64",
"./parallel-qemu-saved/pc-bios/efi-virtio.rom",
"./parallel-qemu-saved/pc-bios/efi-e1000.rom",
"./qemu-img",
"debug.cfg",
]
Expand All @@ -230,8 +246,7 @@ def set_up_folders(self):
else:
link_address = f"{self.get_experiment_folder_address()}/run/{f.split('/')[-1]}"

if not os.path.exists(link_address):
os.system(f"cp -u {f} {link_address}")
os.system(f"cp -u {f} {link_address}")
# TODO turn WormCacheQFlex address into a parameter
# Copy WormCacheQFlex to lib folder, if it doesn't exist we should throw an error
if not os.path.exists(f"./WormCacheQFlex"):
Expand Down Expand Up @@ -354,7 +369,8 @@ def create_experiment_context(
check_period_quantum_coeff: float = 53.0,
use_cd_rom: bool = False,
machine_freq_ghz: float = 2.0, # Default frequency, can be modified later
include_affinity: bool = False
include_affinity: bool = False,
node_number: int = -1,
) -> ExperimentContext:
# assert False
# TODO add how to create experiment name
Expand Down Expand Up @@ -418,7 +434,8 @@ def create_experiment_context(
use_image_directly=use_image_directly,
image_address="", # will be set up during initialization based on other parameters
loadvm_name=loadvm_name,
include_affinity=include_affinity
include_affinity=include_affinity,
node_number=node_number,
)

e.set_up_folders()
Expand Down
19 changes: 17 additions & 2 deletions commands/docker.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,12 +65,19 @@ def cmd(self) -> str:
-v {cwd}/templates:/home/dev/qflex/templates \
-v {cwd}/typer_inputs:/home/dev/qflex/typer_inputs \
-v {cwd}/commands:/home/dev/qflex/commands \
-v {cwd}/flexus:/home/dev/qflex/flexus \
{qflex_args} \
-v {cwd}/partition.py:/home/dev/qflex/partition.py \
-v {cwd}/result.py:/home/dev/qflex/result.py \
-v {cwd}/Makefile:/home/dev/qflex/Makefile \
-v {cwd}/multi-node-scripts/:/home/dev/qflex/multi-node-scripts \
-v {cwd}/parallel-qemu/:/home/dev/qflex/parallel-qemu \
-v {cwd}/qemu/:/home/dev/qflex/qemu \
-v {cwd}/multi-node-web-search/:/home/dev/qflex/multi-node-web-search \
{micro_scripts} \
--security-opt seccomp=unconfined \
--cap-add SYS_PTRACE \
--cap-add=NET_ADMIN --device=/dev/net/tun \
{self.start_directory} \
{commands_mount} {binary_mount} {self.docker_image_name}
"""
Expand Down Expand Up @@ -107,12 +114,20 @@ def cmd(self) -> str:

local_worm_name = f"{self.docker_image_name_with_worm}:{self.version}"
ghcr_worm_name = f"ghcr.io/parsa-epfl/qflex:{self.docker_image_name_with_worm}-{self.version}"

# TODO do some docker renamig
dep_image_name = "qflex-dependencies"
dep_docker_build_cmd = [
f"""docker buildx build -t {dep_image_name} . -f Dockerfile
""",
]


# TODO centeralize the ghcr.io/parsa-epfl/qflex part
if not self.worm_only:
base_image_build_cmd = [
f"""
docker buildx build -t {local_qflex_name} --build-arg MODE={self.build_type} .
docker buildx build -t {local_qflex_name} . -f Dockerfile.qemu.{self.build_type} --build-arg BASE_IMAGE={dep_image_name}
""",
f"docker tag {local_qflex_name} {ghcr_qflex_name}"
]
Expand All @@ -138,7 +153,7 @@ def cmd(self) -> str:
f"docker push {ghcr_worm_name}"
]

base_cmd = base_image_build_cmd
base_cmd = dep_docker_build_cmd + base_image_build_cmd
worm_cmd = worm_image_cmd
if self.push:
base_cmd += base_image_push_cmd
Expand Down
41 changes: 41 additions & 0 deletions commands/multinode.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
import os
from commands import Executor
from .config import ExperimentContext
from commands.qemu import QemuCommonArgParser


class Multi(Executor):

def __init__(self,
experiment_context: ExperimentContext,):
self.experiment_context = experiment_context
self.qemu_common_parser = QemuCommonArgParser(
experiment_context,
)

def cmd(self) -> str:

# boot_cmd = f"""
# gdb --args ./qemu-system-aarch64 \
# {self.qemu_common_parser.get_qemu_base_args()} \
# -icount shift=0,align=off,sleep=off
# """



boot_cmd = f"""
gdb --args ./qemu-system-aarch64 \
{self.qemu_common_parser.get_qemu_base_args()} \
{self.qemu_common_parser.quantum_args()}
"""

# -icount shift=0,align=off,sleep=off


print(f"{boot_cmd}")


return [
f"cd {self.experiment_context.get_experiment_folder_address()}/run",
boot_cmd
]
Loading