diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index 4e0c1c0..0bfe48b 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -24,7 +24,7 @@ jobs: fetch-depth: 0 persist-credentials: false - name: Run TinySemVer - uses: ashvardanian/tinysemver@v2.1.1 + uses: ashvardanian/tinysemver@v3 with: verbose: "true" version-file: "VERSION" @@ -32,14 +32,14 @@ jobs: CMakeLists.txt:^\s*VERSION (\d+\.\d+\.\d+) Cargo.toml:^version = "(\d+\.\d+\.\d+)" README.md:^\s*GIT_TAG v(\d+\.\d+\.\d+) - README.md:^\s*fork_union\s*=\s*"(\d+\.\d+\.\d+)" - README.md:^\s*fork_union\s*=\s*\{\s*version\s*=\s*"(\d+\.\d+\.\d+)" + README.md:^\s*forkunion\s*=\s*"(\d+\.\d+\.\d+)" + README.md:^\s*forkunion\s*=\s*\{\s*version\s*=\s*"(\d+\.\d+\.\d+)" update-major-version-in: | - include/fork_union.hpp:^#define FORK_UNION_VERSION_MAJOR (\d+) + include/forkunion.hpp:^#define FORKUNION_VERSION_MAJOR (\d+) update-minor-version-in: | - include/fork_union.hpp:^#define FORK_UNION_VERSION_MINOR (\d+) + include/forkunion.hpp:^#define FORKUNION_VERSION_MINOR (\d+) update-patch-version-in: | - include/fork_union.hpp:^#define FORK_UNION_VERSION_PATCH (\d+) + include/forkunion.hpp:^#define FORKUNION_VERSION_PATCH (\d+) dry-run: "true" test_ubuntu_gcc: @@ -62,9 +62,15 @@ jobs: - name: Test C++ run: | set -euxo pipefail - build_artifacts/fork_union_test_cpp17 - build_artifacts/fork_union_test_cpp20 - build_artifacts/fork_union_test_cpp23 + build_artifacts/forkunion_test_cpp17 + build_artifacts/forkunion_test_cpp20 + build_artifacts/forkunion_test_cpp23 + + - name: Test C + run: | + set -euxo pipefail + build_artifacts/forkunion_test_c11 + build_artifacts/forkunion_test_c_gcc_nested - name: Set up Rust run: | @@ -91,15 +97,21 @@ jobs: - name: Build C/C++ run: | sudo apt update - sudo apt install -y cmake build-essential clang + sudo apt install -y cmake build-essential clang libblocksruntime-dev cmake -B build_artifacts -D CMAKE_BUILD_TYPE=RelWithDebInfo cmake --build build_artifacts --config RelWithDebInfo - name: Test C++ run: | set -euxo pipefail - build_artifacts/fork_union_test_cpp17 - build_artifacts/fork_union_test_cpp20 - build_artifacts/fork_union_test_cpp23 + build_artifacts/forkunion_test_cpp17 + build_artifacts/forkunion_test_cpp20 + build_artifacts/forkunion_test_cpp23 + + - name: Test C + run: | + set -euxo pipefail + build_artifacts/forkunion_test_c11 + build_artifacts/forkunion_test_c_clang_blocks - name: Set up Rust run: | @@ -128,9 +140,14 @@ jobs: - name: Test C++ run: | set -euxo pipefail - build_artifacts/fork_union_test_cpp17 - build_artifacts/fork_union_test_cpp20 - build_artifacts/fork_union_test_cpp23 + build_artifacts/forkunion_test_cpp17 + build_artifacts/forkunion_test_cpp20 + build_artifacts/forkunion_test_cpp23 + + - name: Test C + run: | + set -euxo pipefail + build_artifacts/forkunion_test_c11 - name: Set up Rust run: | @@ -159,19 +176,23 @@ jobs: brew update brew reinstall cmake llvm@${{ matrix.llvm }} - - name: Configure (CMake) + - name: Build C/C++ run: | cmake -S . -B build_artifacts -D CMAKE_BUILD_TYPE=${{ matrix.config }} -D CMAKE_CXX_COMPILER=$(brew --prefix llvm@${{ matrix.llvm }})/bin/clang++ + cmake --build build_artifacts --config ${{ matrix.config }} --parallel - - name: Build - run: cmake --build build_artifacts --config ${{ matrix.config }} --parallel + - name: Test C++ + run: | + set -euxo pipefail + build_artifacts/forkunion_test_cpp17 + build_artifacts/forkunion_test_cpp20 + build_artifacts/forkunion_test_cpp23 - - name: Run C++ tests + - name: Test C run: | set -euxo pipefail - build_artifacts/fork_union_test_cpp17 - build_artifacts/fork_union_test_cpp20 - build_artifacts/fork_union_test_cpp23 + build_artifacts/forkunion_test_c11 + build_artifacts/forkunion_test_c_clang_blocks test_windows: name: Windows @@ -190,9 +211,13 @@ jobs: - name: Test C++ run: | $ErrorActionPreference = "Stop" - .\build_artifacts\fork_union_test_cpp17.exe - .\build_artifacts\fork_union_test_cpp20.exe - .\build_artifacts\fork_union_test_cpp23.exe + .\build_artifacts\forkunion_test_cpp17.exe + .\build_artifacts\forkunion_test_cpp20.exe + .\build_artifacts\forkunion_test_cpp23.exe + + - name: Test C + run: | + Write-Host "MSVC still lacks ; skipping C test binary" - name: Set up Rust run: | @@ -226,9 +251,13 @@ jobs: - name: Test C++ run: | $ErrorActionPreference = "Stop" - .\build_artifacts\fork_union_test_cpp17.exe - .\build_artifacts\fork_union_test_cpp20.exe - .\build_artifacts\fork_union_test_cpp23.exe + .\build_artifacts\forkunion_test_cpp17.exe + .\build_artifacts\forkunion_test_cpp20.exe + .\build_artifacts\forkunion_test_cpp23.exe + + - name: Test C + run: | + Write-Host "MSVC still lacks ; skipping C test binary" test_i386: name: Cross-compilation (i386) @@ -245,12 +274,16 @@ jobs: - name: Build C++ run: | cmake -B build_i386 -D CMAKE_C_FLAGS=-m32 -D CMAKE_CXX_FLAGS=-m32 -D CMAKE_BUILD_TYPE=RelWithDebInfo - cmake --build build_i386 --target fork_union_test_cpp17 fork_union_test_cpp20 + cmake --build build_i386 --target forkunion_test_cpp17 forkunion_test_cpp20 forkunion_test_c11 - name: Test C++ run: | - build_i386/fork_union_test_cpp17 - build_i386/fork_union_test_cpp20 + build_i386/forkunion_test_cpp17 + build_i386/forkunion_test_cpp20 + + - name: Test C + run: | + build_i386/forkunion_test_c11 test_armhf: name: Cross-compilation (armhf) @@ -266,13 +299,17 @@ jobs: - name: Build C++ run: | - cmake -B build_armhf -D CMAKE_CXX_COMPILER=arm-linux-gnueabihf-g++ -D CMAKE_BUILD_TYPE=RelWithDebInfo - cmake --build build_armhf --target fork_union_test_cpp17 fork_union_test_cpp20 + cmake -B build_armhf -D CMAKE_C_COMPILER=arm-linux-gnueabihf-gcc -D CMAKE_CXX_COMPILER=arm-linux-gnueabihf-g++ -D CMAKE_BUILD_TYPE=RelWithDebInfo + cmake --build build_armhf --target forkunion_test_cpp17 forkunion_test_cpp20 forkunion_test_c11 - name: Test C++ run: | - qemu-arm-static -L /usr/arm-linux-gnueabihf build_armhf/fork_union_test_cpp17 - qemu-arm-static -L /usr/arm-linux-gnueabihf build_armhf/fork_union_test_cpp20 + qemu-arm-static -L /usr/arm-linux-gnueabihf build_armhf/forkunion_test_cpp17 + qemu-arm-static -L /usr/arm-linux-gnueabihf build_armhf/forkunion_test_cpp20 + + - name: Test C + run: | + qemu-arm-static -L /usr/arm-linux-gnueabihf build_armhf/forkunion_test_c11 test_s390x: name: Cross-compilation (s390x) @@ -288,10 +325,14 @@ jobs: - name: Build C++ run: | - cmake -B build_s390x -D CMAKE_CXX_COMPILER=s390x-linux-gnu-g++ -D CMAKE_BUILD_TYPE=RelWithDebInfo - cmake --build build_s390x --target fork_union_test_cpp17 fork_union_test_cpp20 + cmake -B build_s390x -D CMAKE_C_COMPILER=s390x-linux-gnu-gcc -D CMAKE_CXX_COMPILER=s390x-linux-gnu-g++ -D CMAKE_BUILD_TYPE=RelWithDebInfo + cmake --build build_s390x --target forkunion_test_cpp17 forkunion_test_cpp20 forkunion_test_c11 - name: Test C++ run: | - qemu-s390x-static -L /usr/s390x-linux-gnu build_s390x/fork_union_test_cpp17 - qemu-s390x-static -L /usr/s390x-linux-gnu build_s390x/fork_union_test_cpp20 + qemu-s390x-static -L /usr/s390x-linux-gnu build_s390x/forkunion_test_cpp17 + qemu-s390x-static -L /usr/s390x-linux-gnu build_s390x/forkunion_test_cpp20 + + - name: Test C + run: | + qemu-s390x-static -L /usr/s390x-linux-gnu build_s390x/forkunion_test_c11 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 978ffcd..6089cb9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,7 +27,7 @@ jobs: fetch-depth: 0 persist-credentials: false - name: Run TinySemVer - uses: ashvardanian/tinysemver@v2.1.1 + uses: ashvardanian/tinysemver@v3 with: verbose: "true" version-file: "VERSION" @@ -35,14 +35,14 @@ jobs: CMakeLists.txt:^\s*VERSION (\d+\.\d+\.\d+) Cargo.toml:^version = "(\d+\.\d+\.\d+)" README.md:^\s*GIT_TAG v(\d+\.\d+\.\d+) - README.md:^\s*fork_union\s*=\s*"(\d+\.\d+\.\d+)" - README.md:^\s*fork_union\s*=\s*\{\s*version\s*=\s*"(\d+\.\d+\.\d+)" + README.md:^\s*forkunion\s*=\s*"(\d+\.\d+\.\d+)" + README.md:^\s*forkunion\s*=\s*\{\s*version\s*=\s*"(\d+\.\d+\.\d+)" update-major-version-in: | - include/fork_union.hpp:^#define FORK_UNION_VERSION_MAJOR (\d+) + include/forkunion.hpp:^#define FORKUNION_VERSION_MAJOR (\d+) update-minor-version-in: | - include/fork_union.hpp:^#define FORK_UNION_VERSION_MINOR (\d+) + include/forkunion.hpp:^#define FORKUNION_VERSION_MINOR (\d+) update-patch-version-in: | - include/fork_union.hpp:^#define FORK_UNION_VERSION_PATCH (\d+) + include/forkunion.hpp:^#define FORKUNION_VERSION_PATCH (\d+) dry-run: "false" push: "true" create-release: "true" diff --git a/.gitignore b/.gitignore index 8ed97ad..418a891 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,6 @@ # Rust build artifacts Cargo.lock target/ +.zig-cache/ +zig-out/ +zig-cache/ \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index 415f872..e6729a5 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -9,7 +9,7 @@ "type": "cppdbg", "request": "launch", "preLaunchTask": "Build: Debug with GDB", - "program": "${workspaceFolder}/build_debug/fork_union_test_cpp20", + "program": "${workspaceFolder}/build_debug/forkunion_test_cpp20", "cwd": "${workspaceFolder}", "args": [], "setupCommands": [ @@ -40,7 +40,7 @@ "MIMode": "gdb" }, "windows": { - "program": "${workspaceFolder}\\build_debug\\scripts\\fork_union_test_cpp20.exe", + "program": "${workspaceFolder}\\build_debug\\scripts\\forkunion_test_cpp20.exe", "MIMode": "gdb", "miDebuggerPath": "C:\\MinGw\\bin\\gdb.exe" } @@ -50,7 +50,7 @@ "type": "cppdbg", "request": "launch", "preLaunchTask": "Build: Debug i386 with GDB", - "program": "${workspaceFolder}/build_i386/fork_union_test_cpp20", + "program": "${workspaceFolder}/build_i386/forkunion_test_cpp20", "cwd": "${workspaceFolder}", "args": [], "setupCommands": [ @@ -86,7 +86,7 @@ "type": "cppdbg", "request": "launch", "preLaunchTask": "Build: Debug with LLDB", - "program": "${workspaceFolder}/build_debug/fork_union_test_cpp20", + "program": "${workspaceFolder}/build_debug/forkunion_test_cpp20", "cwd": "${workspaceFolder}", "args": [], "setupCommands": [ @@ -116,7 +116,7 @@ "type": "cppdbg", "request": "launch", "preLaunchTask": "Build: Debug with GDB", - "program": "${workspaceFolder}/build_debug/fork_union_nbody", + "program": "${workspaceFolder}/build_debug/forkunion_nbody", "cwd": "${workspaceFolder}", "args": [], "setupCommands": [ @@ -146,7 +146,7 @@ }, { "name": "NBODY_BACKEND", - "value": "fork_union_dynamic" + "value": "forkunion_dynamic" } ], "stopAtEntry": false, diff --git a/.vscode/settings.json b/.vscode/settings.json index 0230450..b6999b3 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -9,28 +9,52 @@ "editor.insertSpaces": true, "editor.tabSize": 4 }, + "[yaml]": { + "editor.detectIndentation": false, + "editor.insertSpaces": true, + "editor.tabSize": 2, + "prettier.tabWidth": 2 + }, + "[yml]": { + "editor.detectIndentation": false, + "editor.insertSpaces": true, + "editor.tabSize": 2, + "prettier.tabWidth": 2 + }, "cSpell.words": [ + "anyopaque", + "anytype", "ashvardanian", + "callconv", "cntfrq", "cntvct", "codegen", "colocations", "combinators", + "comptime", "Condvar", "constexpr", "coprime", "ctest", + "forkunion", "fprintf", "futex", "gethugepagesizes", + "Hashimoto", "hugepages", "HugeTLBfs", "inclusivity", "libnuma", + "libxev", + "Miri", + "MSRV", + "MSVC", "nbody", "noexcept", "NUMA", "OpenMP", + "orelse", + "overaligned", "prefetcher", "println", "pthreads", @@ -43,8 +67,10 @@ "STREQUAL", "SysCall", "SysFS", + "tinysemver", "topo", "TSAN", + "usize", "Vardanian", "vecs", "WFET" diff --git a/CMakeLists.txt b/CMakeLists.txt index 1f5f025..bf4c881 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,21 +12,21 @@ if (POLICY CMP0091) endif () project( - fork_union + forkunion VERSION 2.3.0 DESCRIPTION "Low-latency OpenMP-style NUMA-aware cross-platform fine-grained parallelism library" LANGUAGES CXX ) # Control strict interface warnings for consumers -option(FORK_UNION_STRICT "Propagate strict warnings (including -Werror) to consumers" OFF) +option(FORKUNION_STRICT "Propagate strict warnings (including -Werror) to consumers" OFF) # Control NUMA enablement: AUTO, ON, OFF -set(FORK_UNION_ENABLE_NUMA +set(FORKUNION_ENABLE_NUMA "AUTO" CACHE STRING "NUMA support mode: AUTO, ON, or OFF" ) -set_property(CACHE FORK_UNION_ENABLE_NUMA PROPERTY STRINGS "AUTO;ON;OFF") +set_property(CACHE FORKUNION_ENABLE_NUMA PROPERTY STRINGS "AUTO;ON;OFF") # Enforce C++17 as the minimum standard for the project. set(PROJECT_CXX_STANDARD 17) @@ -34,24 +34,24 @@ set(PROJECT_CXX_EXTENSIONS OFF) set(PROJECT_CXX_STANDARD_REQUIRED ON) # Header-only interface library -add_library(fork_union INTERFACE) +add_library(forkunion INTERFACE) target_include_directories( - fork_union INTERFACE $ $ + forkunion INTERFACE $ $ ) # Set C++17 requirement and features properly for library consumers -target_compile_features(fork_union INTERFACE cxx_std_17) +target_compile_features(forkunion INTERFACE cxx_std_17) set_target_properties( - fork_union + forkunion PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON CXX_EXTENSIONS OFF ) # Strictest possible compilation flags with fatal warnings -if (FORK_UNION_STRICT) +if (FORKUNION_STRICT) target_compile_options( - fork_union + forkunion INTERFACE # GCC/Clang: Maximum warnings + treat warnings as errors + security hardening $<$:-Wall -Wextra @@ -111,21 +111,21 @@ if (FORK_UNION_STRICT) ) endif () -# Pre-compiled libraries built from `c/lib.cpp` -add_library(fork_union_dynamic SHARED c/lib.cpp) -add_library(fork_union_static STATIC c/lib.cpp) +# Pre-compiled libraries built from `c/forkunion.cpp` +add_library(forkunion_dynamic SHARED c/forkunion.cpp) +add_library(forkunion_static STATIC c/forkunion.cpp) # Prefer C++20 for library builds set_target_properties( - fork_union_dynamic fork_union_static + forkunion_dynamic forkunion_static PROPERTIES CXX_STANDARD 20 CXX_STANDARD_REQUIRED ON CXX_EXTENSIONS OFF ) # Re-use the public interface of the header-only target -target_link_libraries(fork_union_dynamic PUBLIC fork_union) -target_link_libraries(fork_union_static PUBLIC fork_union) +target_link_libraries(forkunion_dynamic PUBLIC forkunion) +target_link_libraries(forkunion_static PUBLIC forkunion) # Probe compiler support for -fcf-protection=full (depends on compiler+arch) include(CheckCXXCompilerFlag) @@ -134,7 +134,7 @@ check_cxx_compiler_flag("-fcf-protection=full" HAS_CFP) # Security hardening flags for compiled libraries target_compile_options( - fork_union_dynamic + forkunion_dynamic PRIVATE # Stack protection and buffer overflow detection $<$:-fstack-protector-strong -D_FORTIFY_SOURCE=2> @@ -147,7 +147,7 @@ target_compile_options( /guard:cf> ) target_compile_options( - fork_union_static + forkunion_static PRIVATE $<$:-fstack-protector-strong -D_FORTIFY_SOURCE=2> $<$,$>:-fcf-protection=full> @@ -159,7 +159,7 @@ target_compile_options( # Hardened linking flags target_link_options( - fork_union_dynamic + forkunion_dynamic PRIVATE # Enable RELRO, stack canaries, and NX bit $<$:-Wl,-z,relro,-z,now,-z,noexecstack> @@ -169,7 +169,7 @@ target_link_options( /guard:cf> ) target_link_options( - fork_union_static PRIVATE $<$:-Wl,-z,relro,-z,now,-z,noexecstack> + forkunion_static PRIVATE $<$:-Wl,-z,relro,-z,now,-z,noexecstack> $<$:/guard:cf> ) @@ -225,9 +225,9 @@ endif () # Tests & benchmarking scripts include(CTest) -option(FORK_UNION_BUILD_TESTS "Build fork_union tests" ON) +option(FORKUNION_BUILD_TESTS "Build forkunion tests" ON) -if (BUILD_TESTING AND FORK_UNION_BUILD_TESTS) +if (BUILD_TESTING AND FORKUNION_BUILD_TESTS) enable_testing() add_subdirectory(scripts) endif () @@ -237,14 +237,14 @@ include(GNUInstallDirs) install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) install( - TARGETS fork_union - EXPORT fork_unionTargets + TARGETS forkunion + EXPORT forkunionTargets INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} ) install( - TARGETS fork_union_dynamic fork_union_static - EXPORT fork_unionTargets # same export set + TARGETS forkunion_dynamic forkunion_static + EXPORT forkunionTargets # same export set LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} # .so / .dylib ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} # .a RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} # .dll on Windows @@ -253,28 +253,28 @@ install( # Export config files for find_package include(CMakePackageConfigHelpers) write_basic_package_version_file( - "${CMAKE_CURRENT_BINARY_DIR}/fork_unionConfigVersion.cmake" + "${CMAKE_CURRENT_BINARY_DIR}/forkunionConfigVersion.cmake" VERSION ${PROJECT_VERSION} COMPATIBILITY AnyNewerVersion ) configure_package_config_file( - cmake/fork_unionConfig.cmake.in "${CMAKE_CURRENT_BINARY_DIR}/fork_unionConfig.cmake" - INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/fork_union + cmake/forkunionConfig.cmake.in "${CMAKE_CURRENT_BINARY_DIR}/forkunionConfig.cmake" + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/forkunion ) -install(FILES "${CMAKE_CURRENT_BINARY_DIR}/fork_unionConfig.cmake" - "${CMAKE_CURRENT_BINARY_DIR}/fork_unionConfigVersion.cmake" - DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/fork_union +install(FILES "${CMAKE_CURRENT_BINARY_DIR}/forkunionConfig.cmake" + "${CMAKE_CURRENT_BINARY_DIR}/forkunionConfigVersion.cmake" + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/forkunion ) install( - EXPORT fork_unionTargets - NAMESPACE fork_union:: - DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/fork_union + EXPORT forkunionTargets + NAMESPACE forkunion:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/forkunion ) -# NUMA selection logic controlled by FORK_UNION_ENABLE_NUMA +# NUMA selection logic controlled by FORKUNION_ENABLE_NUMA if (UNIX AND NOT APPLE) # Linux # Find POSIX threads library find_package(Threads) @@ -288,23 +288,23 @@ if (UNIX AND NOT APPLE) # Linux message(STATUS "libNUMA not found - NUMA-dependent features unavailable") endif () - if (FORK_UNION_ENABLE_NUMA STREQUAL "ON") + if (FORKUNION_ENABLE_NUMA STREQUAL "ON") if (Threads_FOUND AND NUMA_FOUND) - target_link_libraries(fork_union INTERFACE ${NUMA_LIBRARY} Threads::Threads) - target_compile_definitions(fork_union INTERFACE FU_ENABLE_NUMA=1) + target_link_libraries(forkunion INTERFACE ${NUMA_LIBRARY} Threads::Threads) + target_compile_definitions(forkunion INTERFACE FU_ENABLE_NUMA=1) else () - message(FATAL_ERROR "FORK_UNION_ENABLE_NUMA=ON, but Threads or libnuma not found") + message(FATAL_ERROR "FORKUNION_ENABLE_NUMA=ON, but Threads or libnuma not found") endif () - elseif (FORK_UNION_ENABLE_NUMA STREQUAL "AUTO") + elseif (FORKUNION_ENABLE_NUMA STREQUAL "AUTO") if (Threads_FOUND AND NUMA_FOUND) - target_link_libraries(fork_union INTERFACE ${NUMA_LIBRARY} Threads::Threads) - target_compile_definitions(fork_union INTERFACE FU_ENABLE_NUMA=1) + target_link_libraries(forkunion INTERFACE ${NUMA_LIBRARY} Threads::Threads) + target_compile_definitions(forkunion INTERFACE FU_ENABLE_NUMA=1) else () - target_compile_definitions(fork_union INTERFACE FU_ENABLE_NUMA=0) + target_compile_definitions(forkunion INTERFACE FU_ENABLE_NUMA=0) endif () else () # OFF - target_compile_definitions(fork_union INTERFACE FU_ENABLE_NUMA=0) + target_compile_definitions(forkunion INTERFACE FU_ENABLE_NUMA=0) endif () else () # Non-Linux platforms - target_compile_definitions(fork_union INTERFACE FU_ENABLE_NUMA=0) + target_compile_definitions(forkunion INTERFACE FU_ENABLE_NUMA=0) endif () diff --git a/Cargo.toml b/Cargo.toml index 2ffbcac..6b1ffc7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,15 +1,15 @@ [package] -name = "fork_union" +name = "forkunion" description = "Low-latency OpenMP-style NUMA-aware cross-platform fine-grained parallelism library" version = "2.3.0" edition = "2021" authors = ["Ash Vardanian"] license = "Apache-2.0" -repository = "https://github.com/ashvardanian/fork_union" +repository = "https://github.com/ashvardanian/forkunion" rust-version = "1.64" # Introduced Core C FFI in stable Rust readme = "README.md" -documentation = "https://docs.rs/fork_union" -homepage = "https://github.com/ashvardanian/fork_union" +documentation = "https://docs.rs/forkunion" +homepage = "https://github.com/ashvardanian/forkunion" keywords = ["numa", "parallel", "thread-pool", "allocator", "no-std"] categories = ["concurrency", "os::linux-apis", "external-ffi-bindings"] include = [ @@ -21,11 +21,11 @@ include = [ "include/**", "c/**", ] -links = "fork_union" +links = "forkunion" [lib] -name = "fork_union" -path = "rust/lib.rs" +name = "forkunion" +path = "rust/forkunion.rs" [build-dependencies] cc = "1.2.40" diff --git a/README.md b/README.md index 1e46e56..11bc501 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# Fork Union ๐Ÿด +# ForkUnion ๐Ÿด -Fork Union is arguably the lowest-latency OpenMP-style NUMA-aware minimalistic scoped thread-pool designed for 'Fork-Join' parallelism in C++, C, and Rust, avoiding ร— [mutexes & system calls](#locks-and-mutexes), ร— [dynamic memory allocations](#memory-allocations), ร— [CAS-primitives](#atomics-and-cas), and ร— [false-sharing](#alignment--false-sharing) of CPU cache-lines on the hot path ๐Ÿด +ForkUnion is arguably the lowest-latency OpenMP-style NUMA-aware minimalistic scoped thread-pool designed for 'Fork-Join' parallelism in C++, C, Rust, and Zig, avoiding ร— [mutexes & system calls](#locks-and-mutexes), ร— [dynamic memory allocations](#memory-allocations), ร— [CAS-primitives](#atomics-and-cas), and ร— [false-sharing](#alignment--false-sharing) of CPU cache-lines on the hot path ๐Ÿด ## Motivation @@ -10,15 +10,15 @@ All of that is slow... and true across C++, C, and Rust projects. Short of [OpenMP](https://en.wikipedia.org/wiki/OpenMP), practically every other solution has high dispatch latency and noticeable memory overhead. OpenMP, however, is not ideal for fine-grained parallelism and is less portable than the C++ and Rust standard libraries. -[![`fork_union` banner](https://github.com/ashvardanian/ashvardanian/blob/master/repositories/fork_union.jpg?raw=true)](https://github.com/ashvardanian/fork_union) +[![`forkunion` banner](https://github.com/ashvardanian/ashvardanian/blob/master/repositories/forkunion.jpg?raw=true)](https://github.com/ashvardanian/forkunion) -This is where __`fork_union`__ comes in. -It's a C++ 17 library with C 99 and Rust bindings ([previously Rust implementation was standalone in v1](#why-not-reimplement-it-in-rust)). +This is where __`forkunion`__ comes in. +It's a C++ 17 library with C 99, Rust, and Zig bindings ([previously Rust implementation was standalone in v1](#why-not-reimplement-it-in-rust)). It supports pinning threads to specific [NUMA](https://en.wikipedia.org/wiki/Non-uniform_memory_access) nodes or individual CPU cores, making it much easier to ensure data locality and halving the latency of individual loads in Big Data applications. ## Basic Usage -__`Fork Union`__ is dead-simple to use! +__`ForkUnion`__ is dead-simple to use! There is no nested parallelism, exception handling, or "future promises"; they are banned. The thread pool itself has a few core operations: @@ -49,21 +49,21 @@ To integrate into your Rust project, add the following lines to Cargo.toml: ```toml [dependencies] -fork_union = "2.3.0" # default -fork_union = { version = "2.3.0", features = ["numa"] } # with NUMA support on Linux +forkunion = "2.3.0" # default +forkunion = { version = "2.3.0", features = ["numa"] } # with NUMA support on Linux ``` Or for the preview development version: ```toml [dependencies] -fork_union = { git = "https://github.com/ashvardanian/fork_union.git", branch = "main-dev" } +forkunion = { git = "https://github.com/ashvardanian/forkunion.git", branch = "main-dev" } ``` A minimal example may look like this: ```rust -use fork_union as fu; +use forkunion as fu; let mut pool = fu::spawn(2); pool.for_threads(|thread_index, colocation_index| { println!("Hello from thread # {} on colocation # {}", thread_index + 1, colocation_index + 1); @@ -91,7 +91,7 @@ A more realistic example with named threads and error handling may look like thi ```rust use std::error::Error; -use fork_union as fu; +use forkunion as fu; fn heavy_math(_: usize) {} @@ -110,33 +110,33 @@ For convenience Rayon-style parallel iterators pull the `prelude` module and [ch ### Intro in C++ -To integrate into your C++ project, either just copy the `include/fork_union.hpp` file into your project, add a Git submodule, or CMake. +To integrate into your C++ project, either just copy the `include/forkunion.hpp` file into your project, add a Git submodule, or CMake. For a Git submodule, run: ```bash -git submodule add https://github.com/ashvardanian/fork_union.git extern/fork_union +git submodule add https://github.com/ashvardanian/forkunion.git extern/forkunion ``` Alternatively, using CMake: ```cmake FetchContent_Declare( - fork_union - GIT_REPOSITORY https://github.com/ashvardanian/fork_union + forkunion + GIT_REPOSITORY https://github.com/ashvardanian/forkunion GIT_TAG v2.3.0 ) -FetchContent_MakeAvailable(fork_union) -target_link_libraries(your_target PRIVATE fork_union::fork_union) +FetchContent_MakeAvailable(forkunion) +target_link_libraries(your_target PRIVATE forkunion::forkunion) ``` Then, include the header in your C++ code: ```cpp -#include // `basic_pool_t` +#include // `basic_pool_t` #include // `stderr` #include // `EXIT_SUCCESS` -namespace fu = ashvardanian::fork_union; +namespace fu = ashvardanian::forkunion; int main() { alignas(fu::default_alignment_k) fu::basic_pool_t pool; @@ -177,7 +177,192 @@ int main() { ``` For advanced usage, refer to the [NUMA section below](#non-uniform-memory-access-numa). -NUMA detection on Linux defaults to AUTO. Override with `-D FORK_UNION_ENABLE_NUMA=ON` or `OFF`. +NUMA detection on Linux defaults to AUTO. Override with `-D FORKUNION_ENABLE_NUMA=ON` or `OFF`. + +### Intro in Zig + +To integrate into your Zig project, add ForkUnion to your `build.zig.zon`: + +```zig +.dependencies = .{ + .forkunion = .{ + .url = "https://github.com/ashvardanian/forkunion/archive/refs/tags/v2.3.0.tar.gz", + .hash = "12200000000000000000000000000000000000000000000000000000000000000000", + }, +}, +``` + +Then import and use in your code: + +```zig +const std = @import("std"); +const fu = @import("forkunion"); + +pub fn main() !void { + var pool = try fu.Pool.init(allocator, 4, .inclusive); + defer pool.deinit(); + + // Execute work on each thread (OpenMP-style parallel) + pool.forThreads(struct { + fn work(thread_idx: usize, colocation_idx: usize) void { + std.debug.print("Thread {}\n", .{thread_idx}); + } + }.work, {}); + + // Distribute 1000 tasks across threads (OpenMP-style parallel for) + var results = [_]i32{0} ** 1000; + pool.forN(1000, struct { + fn process(prong: fu.Prong, ctx: Context) void { + ctx.results[prong.task_index] = @intCast(prong.task_index * 2); + } + }.process, .{ .results = &results }); +} +``` + +Unlike `std.Thread.Pool` task queue for async work, ForkUnion is designed for __data parallelism__ +and __tight parallel loops__ โ€” think OpenMP's `#pragma omp parallel for` with zero allocations on the hot path. + +### Intro in C + +ForkUnion provides a pure C99 API via `forkunion.h`, wrapping the C++ implementation in pre-compiled libraries: `forkunion_static.a` or `forkunion_dynamic.so`. +The C API uses opaque `fu_pool_t` handles and function pointers for callbacks, making it compatible with any C99+ compiler. + +To integrate using CMake: + +```cmake +FetchContent_Declare( + forkunion + GIT_REPOSITORY https://github.com/ashvardanian/forkunion + GIT_TAG v2.3.0 +) +FetchContent_MakeAvailable(forkunion) +target_link_libraries(your_target PRIVATE forkunion::forkunion_static) +``` + +A minimal C example: + +```c +#include // printf +#include // fu_pool_t, fu_pool_new, fu_pool_spawn + +void hello_callback(void *context, size_t thread, size_t colocation) { + (void)context; + printf("Hello from thread %zu (colocation %zu)\n", thread, colocation); +} + +int main(void) { + fu_pool_t *pool = fu_pool_new("my_pool"); + if (!pool || !fu_pool_spawn(pool, fu_count_logical_cores(), fu_caller_inclusive_k)) + return 1; + + fu_pool_for_threads(pool, hello_callback, NULL); + fu_pool_delete(pool); + return 0; +} +``` + +For parallel tasks with context: + +```c +struct task_context { + int *data; + size_t size; +}; + +void process_task(void *ctx, size_t task, size_t thread, size_t colocation) { + (void)thread; (void)colocation; + struct task_context *context = (struct task_context *)ctx; + context->data[task] = task * 2; +} + +int main(void) { + fu_pool_t *pool = fu_pool_new("tasks"); + fu_pool_spawn(pool, 4, fu_caller_inclusive_k); + + int data[100] = {0}; + struct task_context ctx = { .data = data, .size = 100 }; + fu_pool_for_n(pool, 100, process_task, &ctx); // static scheduling + fu_pool_for_n_dynamic(pool, 100, process_task, &ctx); // dynamic scheduling + + fu_pool_delete(pool); + return 0; +} +``` + +#### GCC Nested Functions Extension + +GCC supports [nested functions](https://gcc.gnu.org/onlinedocs/gcc/Nested-Functions.html) that can capture variables from the enclosing scope: + +```c +#include +#include +#include + +int main(void) { + fu_pool_t *pool = fu_pool_new("gcc_nested"); + fu_pool_spawn(pool, 4, fu_caller_inclusive_k); + + atomic_size_t counter = 0; + + // GCC nested function - captures 'counter' from enclosing scope + void nested_callback(void *ctx, size_t task, size_t thread, size_t colocation) { + (void)ctx; (void)thread; (void)colocation; + atomic_fetch_add(&counter, 1); + } + + fu_pool_for_n(pool, 100, nested_callback, NULL); + printf("Completed %zu tasks\n", (size_t)atomic_load(&counter)); + + fu_pool_delete(pool); + return 0; +} +``` + +Compile: `gcc -std=c11 test.c -lforkunion_static -lpthread -lnuma` + +#### Clang Blocks Extension + +Clang provides [blocks](https://clang.llvm.org/docs/BlockLanguageSpec.html) with `^{}` syntax: + +```c +#include +#include +#include +#include + +typedef void (^task_block_t)(void *, size_t, size_t, size_t); + +struct block_wrapper { task_block_t block; }; + +void block_wrapper_fn(void *ctx, size_t task, size_t thread, size_t colocation) { + ((struct block_wrapper *)ctx)->block(NULL, task, thread, colocation); +} + +int main(void) { + fu_pool_t *pool = fu_pool_new("clang_blocks"); + fu_pool_spawn(pool, 4, fu_caller_inclusive_k); + + __block atomic_size_t counter = 0; + + task_block_t my_block = ^(void *c, size_t task, size_t t, size_t col) { + (void)c; (void)t; (void)col; + atomic_fetch_add(&counter, 1); + }; + + task_block_t heap_block = Block_copy(my_block); + struct block_wrapper wrapper = { .block = heap_block }; + + fu_pool_for_n(pool, 100, block_wrapper_fn, &wrapper); + + Block_release(heap_block); + printf("Completed %zu tasks\n", (size_t)atomic_load(&counter)); + + fu_pool_delete(pool); + return 0; +} +``` + +Compile: `clang -std=c11 -fblocks test.c -lforkunion_static -lpthread -lnuma -lBlocksRuntime` ## Alternatives & Differences @@ -186,8 +371,9 @@ Many other thread-pool implementations are more feature-rich but have different - Modern C++: [`taskflow/taskflow`](https://github.com/taskflow/taskflow), [`progschj/ThreadPool`](https://github.com/progschj/ThreadPool), [`bshoshany/thread-pool`](https://github.com/bshoshany/thread-pool) - Traditional C++: [`vit-vit/CTPL`](https://github.com/vit-vit/CTPL), [`mtrebi/thread-pool`](https://github.com/mtrebi/thread-pool) - Rust: [`tokio-rs/tokio`](https://github.com/tokio-rs/tokio), [`rayon-rs/rayon`](https://github.com/rayon-rs/rayon), [`smol-rs/smol`](https://github.com/smol-rs/smol) +- Zig: [`std.Thread.Pool`](https://ziglang.org/documentation/master/std/#std.Thread.Pool) -Those are not designed for the same OpenMP-like use cases as __`fork_union`__. +Those are not designed for the same OpenMP-like use cases as __`forkunion`__. Instead, they primarily focus on task queuing, which requires significantly more work. ### Locks and Mutexes @@ -247,7 +433,7 @@ Because of these rules, padding hot variables to 128 bytes is a conservative but ### Non-Uniform Memory Access (NUMA) Handling NUMA isn't trivial and is only supported on Linux with the help of the [`libnuma` library](https://github.com/numactl/numactl). -It provides the `mbind` interface to pin specific memory regions to particular NUMA nodes, as well as helper functions to query the system topology, which are exposed via the `fork_union::numa_topology` template. +It provides the `mbind` interface to pin specific memory regions to particular NUMA nodes, as well as helper functions to query the system topology, which are exposed via the `forkunion::numa_topology` template. Let's say you are working on a Big Data application, like brute-forcing Vector Search using the [SimSIMD](https://github.com/ashvardanian/simsimd) library on a 2 dual-socket CPU system, similar to [USearch](https://github.com/unum-cloud/usearch/pulls). The first part of that program may be responsible for sharding the incoming stream of data between distinct memory regions. @@ -256,10 +442,10 @@ That part, in our simple example will be single-threaded: ```cpp #include // `std::vector` #include // `std::span` -#include // `linux_numa_allocator`, `numa_topology_t`, `linux_distributed_pool_t` +#include // `linux_numa_allocator`, `numa_topology_t`, `linux_distributed_pool_t` #include // `simsimd_f32_cos`, `simsimd_distance_t` -namespace fu = ashvardanian::fork_union; +namespace fu = ashvardanian::forkunion; using floats_alloc_t = fu::linux_numa_allocator; constexpr std::size_t dimensions = 768; /// Matches most BERT-like models @@ -373,12 +559,12 @@ Works in tight loops. ### Rayon-style Parallel Iterators For Rayon-style ergonomics, use the parallel iterator API with the `prelude`. -Unlike Rayon, Fork Union's parallel iterators don't depend on the global state and allow explicit control over the thread pool and scheduling strategy. +Unlike Rayon, ForkUnion's parallel iterators don't depend on the global state and allow explicit control over the thread pool and scheduling strategy. For statically shaped workloads, the default static scheduling is more efficient: ```rust -use fork_union as fu; -use fork_union::prelude::*; +use forkunion as fu; +use forkunion::prelude::*; let mut pool = fu::spawn(4); let mut data: Vec = (0..1000).collect(); @@ -415,21 +601,50 @@ This easily composes with other iterator adaptors, like `map`, `filter`, and `zi }); ``` -Moreover, each thread can maintain its own scratch space to avoid contention during reductions. -Cache-line alignment via `CacheAligned` prevents false sharing: +For parallel reductions, ForkUnion provides Rayon-like convenience methods with automatic NUMA-aware cache-aligned scratch allocation: + +```rust +let data: Vec = (0..1_000_000).map(|i| i as u64).collect(); + +// Sum all elements with automatic scratch allocation +let total: u64 = (&data[..]) + .into_par_iter() + .with_pool(&mut pool) + .sum(); + +// Count elements matching a predicate +let evens = (&data[..]) + .into_par_iter() + .filter(|&x| x % 2 == 0) + .with_pool(&mut pool) + .count(); + +// Custom reduction (product) +let product = (&data[..]) + .into_par_iter() + .with_pool(&mut pool) + .reduce( + || 1u64, // initial value + |acc, value, _| *acc *= *value, // fold function + |a, b| a * b // combine function + ); +``` + +For manual control over scratch allocation, use `reduce_with_scratch`: ```rust // Cache-line aligned wrapper to prevent false sharing -let mut scratch: Vec> = +let mut scratch: Vec> = (0..pool.threads()).map(|_| CacheAligned(0)).collect(); -(&data[..]) +let total = (&data[..]) .into_par_iter() .with_pool(&mut pool) - .fold_with_scratch(scratch.as_mut_slice(), |acc, value, _prong| { - acc.0 += *value; - }); -let total: usize = scratch.iter().map(|a| a.0).sum(); + .reduce_with_scratch( + scratch.as_mut_slice(), + |acc, value, _| acc.0 += *value, // fold + |a, b| a.0 += b.0 // combine in-place + ); ``` ## Performance @@ -443,15 +658,15 @@ Additional NUMA-aware Search examples are available in `scripts/search.rs`. C++ benchmarking results for $N=128$ bodies and $I=1e6$ iterations: -| Machine | OpenMP (D) | OpenMP (S) | Fork Union (D) | Fork Union (S) | -| :------------- | ---------: | ---------: | -------------: | -------------: | -| 16x Intel SPR | 18.9s | 12.4s | 16.8s | 8.7s | -| 12x Apple M2 | 1m:34.8s ยฒ | 1m:25.9s ยฒ | 31.5s | 20.3s | -| 96x Graviton 4 | 32.2s | 20.8s | 39.8s | 26.0s | +| Machine | OpenMP (D) | OpenMP (S) | ForkUnion (D) | ForkUnion (S) | +| :------------- | ---------: | ---------: | ------------: | ------------: | +| 16x Intel SPR | 18.9s | 12.4s | 16.8s | 8.7s | +| 12x Apple M2 | 1m:34.8s ยฒ | 1m:25.9s ยฒ | 31.5s | 20.3s | +| 96x Graviton 4 | 32.2s | 20.8s | 39.8s | 26.0s | Rust benchmarking results for $N=128$ bodies and $I=1e6$ iterations: -| Machine | Rayon (D) | Rayon (S) | Fork Union (D) | Fork Union (S) | +| Machine | Rayon (D) | Rayon (S) | ForkUnion (D) | ForkUnion (S) | | :------------- | ---------: | ---------: | -------------: | -------------: | | 16x Intel SPR | ๐Ÿ”„ 45.4s | ๐Ÿ”„ 32.1s | 18.1s, ๐Ÿ”„ 22.4s | 12.4s, ๐Ÿ”„ 12.9s | | 12x Apple M2 | ๐Ÿ”„ 1m:47.8s | ๐Ÿ”„ 1m:07.1s | 24.5s, ๐Ÿ”„ 26.8s | 11.0s, ๐Ÿ”„ 11.8s | @@ -459,15 +674,26 @@ Rust benchmarking results for $N=128$ bodies and $I=1e6$ iterations: > ยน Another common workload is "Parallel Reductions" covered in a separate [repository](https://github.com/ashvardanian/ParallelReductionsBenchmark). > ยฒ When a combination of performance and efficiency cores is used, dynamic stealing may be more efficient than static slicing. It's also fair to say, that OpenMP is not optimized for AppleClang. -> ๐Ÿ”„ Rotation emoji stands for iterators, the default way to use Rayon and the opt-in slower, but more convenient variant for Fork Union. +> ๐Ÿ”„ Rotation emoji stands for iterators, the default way to use Rayon and the opt-in slower, but more convenient variant for ForkUnion. + +Zig benchmarking results for $N=128$ bodies and $I=1e6$ iterations: + +| Machine | Standard (S) | ForkUnion (D) | ForkUnion (S) | +| :------------- | -----------: | ------------: | ------------: | +| 16x Intel SPR | 2m52.0s | 18.2s | 12.8s | +| 12x Apple M2 | 1m44.8s | 33.2s | 12.2s | +| 96x Graviton 4 | - | - | - | + +> Benchmarking suite also includes [Spice](https://github.com/judofyr/spice) and [libXEV](https://github.com/mitchellh/libxev), two popular Zig libraries for async processing, but those don't provide comparable bulk-synchronous APIs. +> Thus, typically, all of the submitted tasks are executed on a single thread, making results not comparable. You can rerun those benchmarks with the following commands: ```bash cmake -B build_release -D CMAKE_BUILD_TYPE=Release cmake --build build_release --config Release -time NBODY_COUNT=128 NBODY_ITERATIONS=1000000 NBODY_BACKEND=fork_union_static build_release/fork_union_nbody -time NBODY_COUNT=128 NBODY_ITERATIONS=1000000 NBODY_BACKEND=fork_union_dynamic build_release/fork_union_nbody +time NBODY_COUNT=128 NBODY_ITERATIONS=1000000 NBODY_BACKEND=forkunion_static build_release/forkunion_nbody +time NBODY_COUNT=128 NBODY_ITERATIONS=1000000 NBODY_BACKEND=forkunion_dynamic build_release/forkunion_nbody ``` > Consult the header of `scripts/nbody.cpp` and `scripts/nbody.rs` for additional benchmarking options. @@ -516,7 +742,7 @@ To run the C++ tests, use CMake: cmake -B build_release -D CMAKE_BUILD_TYPE=Release -D BUILD_TESTING=ON cmake --build build_release --config Release -j ctest --test-dir build_release # run all tests -build_release/fork_union_nbody # run the benchmarks +build_release/forkunion_nbody # run the benchmarks ``` For C++ debug builds, consider using the VS Code debugger presets or the following commands: @@ -524,7 +750,7 @@ For C++ debug builds, consider using the VS Code debugger presets or the followi ```bash cmake -B build_debug -D CMAKE_BUILD_TYPE=Debug -D BUILD_TESTING=ON cmake --build build_debug --config Debug # build with Debug symbols -build_debug/fork_union_test_cpp20 # run a single test executable +build_debug/forkunion_test_cpp20 # run a single test executable ``` To run static analysis: @@ -549,7 +775,7 @@ To build with an alternative compiler, like LLVM Clang, use the following comman sudo apt-get install libomp-15-dev clang++-15 # OpenMP version must match Clang cmake -B build_debug -D CMAKE_BUILD_TYPE=Debug -D CMAKE_CXX_COMPILER=clang++-15 cmake --build build_debug --config Debug -build_debug/fork_union_test_cpp20 +build_debug/forkunion_test_cpp20 ``` Or on macOS with Apple Clang: @@ -558,19 +784,29 @@ Or on macOS with Apple Clang: brew install llvm@20 cmake -B build_debug -D CMAKE_BUILD_TYPE=Debug -D CMAKE_CXX_COMPILER=$(brew --prefix llvm@20)/bin/clang++ cmake --build build_debug --config Debug -build_debug/fork_union_test_cpp20 +build_debug/forkunion_test_cpp20 ``` -For Rust, use the following command: +For Rust, use the following commands to verify no_std compatibility: ```bash -rustup toolchain install # for Alloc API -cargo miri test # to catch UBs -cargo build --features numa # for NUMA support on Linux -cargo test --release # to run the tests fast -cargo test --features numa --release # for NUMA tests on Linux +rustup toolchain install +cargo build --lib --no-default-features --release +cargo build --lib --no-default-features --features numa --release +cargo doc --lib --no-default-features --no-deps ``` +Verify the tests pass: + +```bash +cargo test --lib --release +cargo test --doc --release +cargo test --lib --features numa --release +``` + +Rust provides a lot of tooling for concurrency testing, like Miri and Loom. +Most of it, however, is not applicable in this case, as the core logic is implemented in C++. + To automatically detect the Minimum Supported Rust Version (MSRV): ```sh @@ -578,6 +814,23 @@ cargo +stable install cargo-msrv cargo msrv find --ignore-lockfile ``` +--- + +For Zig, use the following commands: + +```bash +zig build test --summary all # run tests +zig build -Dnuma=true # enable NUMA support (Linux) + +# Run benchmark from the `scripts` directory +cd scripts +zig build -Doptimize=ReleaseFast +time NBODY_COUNT=128 NBODY_ITERATIONS=1000000 NBODY_BACKEND=forkunion_static \ + ./zig-out/bin/nbody_zig +``` + +Check the `scripts/nbody.zig` header for additional benchmarking options. + ## License Licensed under the Apache License, Version 2.0. See `LICENSE` for details. diff --git a/build.rs b/build.rs index f42620b..1cb2776 100644 --- a/build.rs +++ b/build.rs @@ -9,7 +9,7 @@ fn main() -> Result<(), cc::Error> { build .cpp(true) // Enable C++ support .std("c++17") // Use C++17 standard - .file("c/lib.cpp") + .file("c/forkunion.cpp") .include("include") .define("FU_ENABLE_NUMA", if enable_numa { "1" } else { "0" }) .opt_level(2) // Optimize compiled C++ to -O2 @@ -17,15 +17,15 @@ fn main() -> Result<(), cc::Error> { .warnings(false); // Compile the C++ library first, so Cargo emits - // `-lstatic=fork_union` before we add dependent libs. - if let Err(e) = build.try_compile("fork_union") { + // `-lstatic=forkunion` before we add dependent libs. + if let Err(e) = build.try_compile("forkunion") { print!("cargo:warning={e}"); return Err(e); } // Important: add dependent system libraries AFTER the static lib. // For GNU ld, static libraries are resolved left-to-right, so - // `-lnuma -lpthread` must appear after `-lfork_union` to satisfy symbols. + // `-lnuma -lpthread` must appear after `-lforkunion` to satisfy symbols. if enable_numa { // Link against `libnuma` when NUMA is enabled on Linux println!("cargo:rustc-link-lib=numa"); @@ -36,9 +36,9 @@ fn main() -> Result<(), cc::Error> { println!("cargo:rustc-link-lib=pthread"); } - println!("cargo:rerun-if-changed=c/lib.cpp"); - println!("cargo:rerun-if-changed=rust/lib.rs"); - println!("cargo:rerun-if-changed=include/fork_union.h"); - println!("cargo:rerun-if-changed=include/fork_union.hpp"); + println!("cargo:rerun-if-changed=c/forkunion.cpp"); + println!("cargo:rerun-if-changed=rust/forkunion.rs"); + println!("cargo:rerun-if-changed=include/forkunion.h"); + println!("cargo:rerun-if-changed=include/forkunion.hpp"); Ok(()) } diff --git a/build.zig b/build.zig new file mode 100644 index 0000000..41d66a1 --- /dev/null +++ b/build.zig @@ -0,0 +1,80 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +pub fn build(b: *std.Build) void { + // Check Zig version compatibility (requires 0.15.0 or later) + if (builtin.zig_version.major == 0 and builtin.zig_version.minor < 15) { + @panic("ForkUnion requires Zig 0.15.0 or later. Please upgrade your Zig toolchain."); + } + + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + // Determine NUMA support + const enable_numa = b.option(bool, "numa", "Enable NUMA support (Linux only)") orelse + (target.result.os.tag == .linux); + + // Compile the C++ library from c/forkunion.cpp (like Rust's build.rs does) + const lib = b.addLibrary(.{ + .name = "forkunion", + .linkage = .static, + .root_module = b.createModule(.{ + .target = target, + .optimize = optimize, + }), + }); + + // Build C++ flags + const cpp_flags = if (enable_numa and target.result.os.tag == .linux) + &[_][]const u8{ + "-std=c++20", + "-fno-exceptions", + "-fno-rtti", + "-DFU_ENABLE_NUMA=1", + } + else + &[_][]const u8{ + "-std=c++20", + "-fno-exceptions", + "-fno-rtti", + "-DFU_ENABLE_NUMA=0", + }; + + lib.addCSourceFile(.{ + .file = b.path("c/forkunion.cpp"), + .flags = cpp_flags, + }); + + lib.addIncludePath(b.path("include")); + lib.linkLibCpp(); // Use Zig's bundled `libc++` instead of system `libstdc++` + + b.installArtifact(lib); + + // Create forkunion module for use as a dependency + _ = b.addModule("forkunion", .{ + .root_source_file = b.path("zig/forkunion.zig"), + .target = target, + }); + + // Unit tests + const test_step = b.step("test", "Run library tests"); + const lib_tests = b.addTest(.{ + .root_module = b.createModule(.{ + .root_source_file = b.path("zig/forkunion.zig"), + .target = target, + .optimize = optimize, + }), + }); + + lib_tests.addIncludePath(b.path("include")); + lib_tests.linkLibrary(lib); + if (target.result.os.tag == .linux) { + lib_tests.root_module.linkSystemLibrary("pthread", .{}); + if (enable_numa) { + lib_tests.root_module.linkSystemLibrary("numa", .{}); + } + } + + const run_tests = b.addRunArtifact(lib_tests); + test_step.dependOn(&run_tests.step); +} diff --git a/build.zig.zon b/build.zig.zon new file mode 100644 index 0000000..5417ca3 --- /dev/null +++ b/build.zig.zon @@ -0,0 +1,16 @@ +.{ + .name = .forkunion, + .version = "2.3.0", + .fingerprint = 0xc31742f3e89a27c7, + .minimum_zig_version = "0.15.0", + .paths = .{ + "build.zig", + "build.zig.zon", + "zig/", + "include/", + "c/", + "scripts/", + "README.md", + "LICENSE", + }, +} diff --git a/c/lib.cpp b/c/forkunion.cpp similarity index 93% rename from c/lib.cpp rename to c/forkunion.cpp index 0c65a7b..e1d7baf 100644 --- a/c/lib.cpp +++ b/c/forkunion.cpp @@ -1,11 +1,11 @@ /** * @brief Low-latency OpenMP-style NUMA-aware cross-platform fine-grained parallelism library. - * @file lib.cpp + * @file forkunion.cpp * @author Ash Vardanian * @date June 27, 2025 */ -#include // C type aliases -#include // C++ core implementation +#include // C type aliases +#include // C++ core implementation #include // `std::in_place_type_t` #include // `std::max` @@ -13,7 +13,7 @@ #include // `std::uint8_t` #include // `std::aligned_storage` -namespace fu = ashvardanian::fork_union; +namespace fu = ashvardanian::forkunion; using thread_allocator_t = std::allocator; @@ -22,7 +22,7 @@ using thread_allocator_t = std::allocator; * * MSVC cannot handle alignas > 64 when objects are passed by value in `std::variant`. * This custom implementation uses a tagged union with manual type management. - * @see https://github.com/ashvardanian/fork_union/issues/26 + * @see https://github.com/ashvardanian/forkunion/issues/26 */ struct pool_variants_t { @@ -269,9 +269,9 @@ bool globals_initialize(void) { extern "C" { -int fu_version_major(void) { return FORK_UNION_VERSION_MAJOR; } -int fu_version_minor(void) { return FORK_UNION_VERSION_MINOR; } -int fu_version_patch(void) { return FORK_UNION_VERSION_PATCH; } +int fu_version_major(void) { return FORKUNION_VERSION_MAJOR; } +int fu_version_minor(void) { return FORKUNION_VERSION_MINOR; } +int fu_version_patch(void) { return FORKUNION_VERSION_PATCH; } int fu_enabled_numa(void) { return FU_ENABLE_NUMA; } #pragma region - Metadata @@ -388,17 +388,47 @@ void fu_free(FU_MAYBE_UNUSED_ size_t numa_node_index, void *pointer, FU_MAYBE_UN #pragma region - Lifetime +/** + * @brief Cross-platform aligned memory allocation. + * @note Returns nullptr on failure, never throws exceptions. + */ +inline void *fu_aligned_malloc(std::size_t size, std::size_t alignment) noexcept { +#if defined(_MSC_VER) + return _aligned_malloc(size, alignment); +#elif defined(__unix__) || defined(__unix) || defined(unix) || defined(__APPLE__) + void *ptr = nullptr; + return (posix_memalign(&ptr, alignment, size) == 0) ? ptr : nullptr; +#else + return ::operator new(size, std::align_val_t {alignment}, std::nothrow); +#endif +} + +/** + * @brief Cross-platform aligned memory deallocation. + * @note Matches fu_aligned_malloc - must use same alignment value. + */ +inline void fu_aligned_free(void *ptr, std::size_t alignment) noexcept { +#if defined(_MSC_VER) + _aligned_free(ptr); +#elif defined(__unix__) || defined(__unix) || defined(unix) || defined(__APPLE__) + std::free(ptr); +#else + ::operator delete(ptr, std::align_val_t {alignment}, std::nothrow); +#endif +} + fu_pool_t *fu_pool_new(FU_MAYBE_UNUSED_ char const *name) { if (!globals_initialize()) return nullptr; - opaque_pool_t *opaque = static_cast(std::malloc(sizeof(opaque_pool_t))); + opaque_pool_t *opaque = + static_cast(fu_aligned_malloc(sizeof(opaque_pool_t), alignof(opaque_pool_t))); if (!opaque) return nullptr; // Best case, use the NUMA-aware distributed pool #if FU_ENABLE_NUMA fu::numa_topology_t copied_topology; if (!copied_topology.try_assign(global_numa_topology)) { - std::free(opaque); + fu_aligned_free(opaque, alignof(opaque_pool_t)); return nullptr; } @@ -485,7 +515,7 @@ void fu_pool_delete(fu_pool_t *pool) { // Call the object's destructor and deallocate the memory opaque->~opaque_pool_t(); - std::free(opaque); + fu_aligned_free(opaque, alignof(opaque_pool_t)); } fu_bool_t fu_pool_spawn(fu_pool_t *pool, size_t threads, fu_caller_exclusivity_t c_exclusivity) { diff --git a/cmake/fork_unionConfig.cmake.in b/cmake/fork_unionConfig.cmake.in index f5d8272..2e8b9fc 100644 --- a/cmake/fork_unionConfig.cmake.in +++ b/cmake/fork_unionConfig.cmake.in @@ -1,13 +1,13 @@ @PACKAGE_INIT@ -include("${CMAKE_CURRENT_LIST_DIR}/fork_unionTargets.cmake") +include("${CMAKE_CURRENT_LIST_DIR}/forkunionTargets.cmake") # Provide an un-namespaced alias so downstream consumers can do both: # ~~~ -# target_link_libraries(my_executable PRIVATE fork_union) -# target_link_libraries(my_executable PRIVATE fork_union::fork_union) +# target_link_libraries(my_executable PRIVATE forkunion) +# target_link_libraries(my_executable PRIVATE forkunion::forkunion) # ~~~ -if (NOT TARGET fork_union::fork_union) - add_library(fork_union::fork_union ALIAS fork_union) +if (NOT TARGET forkunion::forkunion) + add_library(forkunion::forkunion ALIAS forkunion) endif () -set(fork_union_VERSION @PACKAGE_VERSION@) +set(forkunion_VERSION @PACKAGE_VERSION@) diff --git a/include/fork_union.h b/include/forkunion.h similarity index 98% rename from include/fork_union.h rename to include/forkunion.h index 4fd295e..7aa170a 100644 --- a/include/fork_union.h +++ b/include/forkunion.h @@ -1,10 +1,10 @@ /** * @brief Low-latency OpenMP-style NUMA-aware cross-platform fine-grained parallelism library. - * @file fork_union.h + * @file forkunion.h * @author Ash Vardanian * @date June 17, 2025 * - * Fork Union provides a minimalistic cross-platform thread-pool implementation and Parallel Algorithms, + * ForkUnion provides a minimalistic cross-platform thread-pool implementation and Parallel Algorithms, * avoiding dynamic memory allocations, exceptions, system calls, and heavy Compare-And-Swap instructions. * The library leverages the "weak memory model" to allow Arm and IBM Power CPUs to aggressively optimize * execution at runtime. It also aggressively tests against overflows on smaller index types, and is safe @@ -13,7 +13,7 @@ * @code{.c} * #include // `printf` * #include // `EXIT_FAILURE`, `EXIT_SUCCESS` - * #include // `fu_pool_t` + * #include // `fu_pool_t` * * struct print_args_context_t { * size_t argc; // ? Number of arguments @@ -30,9 +30,9 @@ * int main(int argc, char *argv[]) { * char const *caps = fu_capabilities_string(); * if (!caps) return EXIT_FAILURE; // ! Thread pool is not supported - * printf("Fork Union capabilities: %s\n", caps); + * printf("ForkUnion capabilities: %s\n", caps); * - * fu_pool_t *pool = fu_pool_new("fork_union_demo"); + * fu_pool_t *pool = fu_pool_new("forkunion_demo"); * if (!pool) return EXIT_FAILURE; // ! Failed to create a thread pool * * size_t threads = fu_count_logical_cores(); @@ -87,9 +87,9 @@ extern "C" { #include // `size_t`, `bool` -int fu_version_major(void); // ? Returns the major version of the Fork Union library -int fu_version_minor(void); // ? Returns the minor version of the Fork Union library -int fu_version_patch(void); // ? Returns the patch version of the Fork Union library +int fu_version_major(void); // ? Returns the major version of the ForkUnion library +int fu_version_minor(void); // ? Returns the minor version of the ForkUnion library +int fu_version_patch(void); // ? Returns the patch version of the ForkUnion library int fu_enabled_numa(void); // ? Checks if the library was compiled with NUMA support #pragma region - Types diff --git a/include/fork_union.hpp b/include/forkunion.hpp similarity index 98% rename from include/fork_union.hpp rename to include/forkunion.hpp index cf2ab2f..7bb8fa2 100644 --- a/include/fork_union.hpp +++ b/include/forkunion.hpp @@ -1,10 +1,10 @@ /** * @brief Low-latency OpenMP-style NUMA-aware cross-platform fine-grained parallelism library. - * @file fork_union.hpp + * @file forkunion.hpp * @author Ash Vardanian * @date May 2, 2025 * - * Fork Union provides a minimalistic cross-platform thread-pool implementation and Parallel Algorithms, + * ForkUnion provides a minimalistic cross-platform thread-pool implementation and Parallel Algorithms, * avoiding dynamic memory allocations, exceptions, system calls, and heavy Compare-And-Swap instructions. * The library leverages the "weak memory model" to allow Arm and IBM Power CPUs to aggressively optimize * execution at runtime. It also aggressively tests against overflows on smaller index types, and is safe @@ -13,9 +13,9 @@ * @code{.cpp} * #include // `std::printf` * #include // `EXIT_FAILURE`, `EXIT_SUCCESS` - * #include // `fu::basic_pool_t` + * #include // `fu::basic_pool_t` * - * using fu = ashvardanian::fork_union; + * using fu = ashvardanian::forkunion; * int main(int argc, char *argv[]) { * * fu::basic_pool_t pool; @@ -78,13 +78,17 @@ #include // `std::hardware_destructive_interference_size` #include // `std::array` -#define FORK_UNION_VERSION_MAJOR 2 -#define FORK_UNION_VERSION_MINOR 3 -#define FORK_UNION_VERSION_PATCH 0 +#define FORKUNION_VERSION_MAJOR 2 +#define FORKUNION_VERSION_MINOR 3 +#define FORKUNION_VERSION_PATCH 0 #if !defined(FU_ALLOW_UNSAFE) +#if defined(__cpp_exceptions) || defined(__EXCEPTIONS) +#define FU_ALLOW_UNSAFE 1 +#else #define FU_ALLOW_UNSAFE 0 #endif +#endif /** * We auto-enable NUMA in Linux builds with GLibC 2.30+ due to `gettid` support. @@ -202,7 +206,7 @@ #endif namespace ashvardanian { -namespace fork_union { +namespace forkunion { #pragma region - Helpers and Constants @@ -292,7 +296,7 @@ enum capabilities_t : unsigned int { }; inline capabilities_t operator|(capabilities_t a, capabilities_t b) { - return static_cast(static_cast(a) | static_cast(b)); + return static_cast(static_cast(a) | static_cast(b)); } struct standard_yield_t { @@ -584,6 +588,19 @@ class unique_padded_buffer { */ struct dummy_lambda_t {}; +template +struct yield_traits { + static constexpr bool supports_no_arg = std::is_nothrow_invocable_r_v; + static constexpr bool supports_thread_index = std::is_nothrow_invocable_r_v; + static constexpr bool valid = supports_no_arg || supports_thread_index; +}; + +template +inline void call_yield_(yield_type_ &yield, thread_index_type_ thread_index) noexcept { + if constexpr (yield_traits::supports_thread_index) { yield(thread_index); } + else { yield(); } +} + /** * @brief A trivial minimalistic lock-free "mutex" implementation using `std::atomic_flag`. * @tparam micro_yield_type_ The type of the yield function to be used for busy-waiting. @@ -606,7 +623,7 @@ class spin_mutex { public: void lock() noexcept { micro_yield_t micro_yield; - while (flag_.test_and_set(std::memory_order_acquire)) micro_yield(); + while (flag_.test_and_set(std::memory_order_acquire)) call_yield_(micro_yield); } bool try_lock() noexcept { return !flag_.test_and_set(std::memory_order_acquire); } void unlock() noexcept { flag_.clear(std::memory_order_release); } @@ -630,7 +647,7 @@ class spin_mutex { public: void lock() noexcept { micro_yield_t micro_yield; - while (flag_.exchange(true, std::memory_order_acquire)) micro_yield(); + while (flag_.exchange(true, std::memory_order_acquire)) call_yield_(micro_yield); } bool try_lock() noexcept { return !flag_.exchange(true, std::memory_order_acquire); } void unlock() noexcept { flag_.store(false, std::memory_order_release); } @@ -969,9 +986,9 @@ constexpr bool can_be_for_slice_callback() noexcept { * @code{.cpp} * #include // `std::printf` * #include // `EXIT_FAILURE`, `EXIT_SUCCESS` - * #include // `basic_pool_t` + * #include // `basic_pool_t` * - * using fu = ashvardanian::fork_union; + * using fu = ashvardanian::forkunion; * int main() { * fu::basic_pool_t pool; // ? Alias to `fu::basic_pool<>` template * if (!pool.try_spawn(std::thread::hardware_concurrency())) return EXIT_FAILURE; @@ -987,9 +1004,9 @@ constexpr bool can_be_for_slice_callback() noexcept { * @code{.cpp} * #include // `std::printf` * #include // `EXIT_FAILURE`, `EXIT_SUCCESS` - * #include // `basic_pool_t` + * #include // `basic_pool_t` * - * using fu = ashvardanian::fork_union; + * using fu = ashvardanian::forkunion; * int main() { * fu::basic_pool_t first_pool, second_pool; * if (!first_pool.try_spawn(2) || !second_pool.try_spawn(2, fu::caller_exclusive_k)) return EXIT_FAILURE; @@ -1018,8 +1035,6 @@ class basic_pool { public: using allocator_t = allocator_type_; using micro_yield_t = micro_yield_type_; - static_assert(std::is_nothrow_invocable_r::value, - "Yield must be callable w/out arguments & return void"); static constexpr std::size_t alignment_k = alignment_; static_assert(is_power_of_two(alignment_k), "Alignment must be a power of 2"); @@ -1035,6 +1050,9 @@ class basic_pool { using punned_fork_context_t = void *; // ? Pointer to the on-stack lambda using trampoline_t = void (*)(punned_fork_context_t, thread_index_t); // ? Wraps lambda's `operator()` + using micro_yield_traits_t = yield_traits; + static_assert(micro_yield_traits_t::valid, "Yield must be invocable w/out args or with a thread index"); + private: // Thread-pool-specific variables: allocator_t allocator_ {}; @@ -1141,20 +1159,33 @@ class basic_pool { // Initializing the thread pool can fail for all kinds of reasons, // that the `std::thread` documentation describes as "implementation-defined". // https://en.cppreference.com/w/cpp/thread/thread/thread - for (thread_index_t i = 0; i < worker_threads; ++i) { + auto spawn_worker = [&](thread_index_t i) noexcept -> bool { + thread_index_t const i_with_caller = i + use_caller_thread; +#if FU_ALLOW_UNSAFE try { - thread_index_t const i_with_caller = i + use_caller_thread; new (&workers[i]) std::thread([this, i_with_caller] { _worker_loop(i_with_caller); }); + return true; } catch (...) { - mood_.store(mood_t::die_k, std::memory_order_release); - for (thread_index_t j = 0; j < i; ++j) { - workers[j].join(); // ? Wait for the thread to exit - workers[j].~thread(); - } - reset_on_failure(); return false; } +#else + new (&workers[i]) std::thread([this, i_with_caller] { _worker_loop(i_with_caller); }); + return true; +#endif + }; + + for (thread_index_t i = 0; i < worker_threads; ++i) { + if (spawn_worker(i)) continue; + + // ! Failed to spawn a thread, roll back everything + mood_.store(mood_t::die_k, std::memory_order_release); + for (thread_index_t j = 0; j < i; ++j) { + workers[j].join(); // ? Wait for the thread to exit + workers[j].~thread(); + } + reset_on_failure(); + return false; } return true; @@ -1217,7 +1248,8 @@ class basic_pool { // Actually wait for everyone to finish micro_yield_t micro_yield; - while (threads_to_sync_.load(std::memory_order_acquire)) micro_yield(); + while (threads_to_sync_.load(std::memory_order_acquire)) + call_yield_(micro_yield, static_cast(0)); } #pragma endregion Core API @@ -1403,7 +1435,7 @@ class basic_pool { micro_yield_t micro_yield; while ((new_epoch = epoch_.load(std::memory_order_acquire)) == last_epoch && (mood = mood_.load(std::memory_order_acquire)) == mood_t::grind_k) - micro_yield(); + call_yield_(micro_yield, thread_index); if (fu_unlikely_(mood == mood_t::die_k)) break; if (fu_unlikely_(mood == mood_t::chill_k) && (new_epoch == last_epoch)) { @@ -2517,8 +2549,6 @@ struct linux_colocated_pool { public: using allocator_t = linux_numa_allocator_t; using micro_yield_t = micro_yield_type_; - static_assert(std::is_nothrow_invocable_r::value, - "Yield must be callable w/out arguments & return void"); static constexpr std::size_t alignment_k = alignment_; static_assert(alignment_k > 0 && (alignment_k & (alignment_k - 1)) == 0, "Alignment must be a power of 2"); @@ -2532,6 +2562,9 @@ struct linux_colocated_pool { using punned_fork_context_t = void *; // ? Pointer to the on-stack lambda using trampoline_t = void (*)(punned_fork_context_t, colocated_thread_t); // ? Wraps lambda's `operator()` + using micro_yield_traits_t = yield_traits; + static_assert(micro_yield_traits_t::valid, "Yield must be invocable w/out args or with a thread index"); + private: using allocator_traits_t = std::allocator_traits; using numa_pthread_allocator_t = typename allocator_traits_t::template rebind_alloc; @@ -2572,9 +2605,9 @@ struct linux_colocated_pool { linux_colocated_pool &operator=(linux_colocated_pool &&) = delete; linux_colocated_pool &operator=(linux_colocated_pool const &) = delete; - explicit linux_colocated_pool(char const *name = "fork_union") noexcept { + explicit linux_colocated_pool(char const *name = "forkunion") noexcept { // Accept NULL or empty names by falling back to a sensible default - char const *effective_name = (name && name[0] != '\0') ? name : "fork_union"; + char const *effective_name = (name && name[0] != '\0') ? name : "forkunion"; std::strncpy(name_, effective_name, sizeof(name_) - 1); name_[sizeof(name_) - 1] = '\0'; } @@ -2861,7 +2894,8 @@ struct linux_colocated_pool { // Actually wait for everyone to finish micro_yield_t micro_yield; - while (threads_to_sync_.load(std::memory_order_acquire)) micro_yield(); + while (threads_to_sync_.load(std::memory_order_acquire)) + call_yield_(micro_yield, static_cast(0)); } #pragma endregion Core API @@ -3071,7 +3105,9 @@ struct linux_colocated_pool { // so spin-loop for a bit until the pool is ready. mood_t mood; micro_yield_t micro_yield; - while ((mood = pool->mood_.load(std::memory_order_acquire)) == mood_t::chill_k) micro_yield(); + while ((mood = pool->mood_.load(std::memory_order_acquire)) == mood_t::chill_k) + // Technically, we are not on the zero thread index, but we don't know our index yet. + call_yield_(micro_yield, static_cast(0)); // If we are ready to start grinding, export this threads metadata to make it externally // observable and controllable. @@ -3106,7 +3142,7 @@ struct linux_colocated_pool { // Wait for either: a new ticket or a stop flag while ((new_epoch = pool->epoch_.load(std::memory_order_acquire)) == last_epoch && (mood = pool->mood_.load(std::memory_order_acquire)) == mood_t::grind_k) - micro_yield(); + call_yield_(micro_yield, global_thread_index); if (fu_unlikely_(mood == mood_t::die_k)) break; if (fu_unlikely_(mood == mood_t::chill_k) && (new_epoch == last_epoch)) { @@ -3350,12 +3386,11 @@ struct linux_distributed_pool { linux_distributed_pool &operator=(linux_distributed_pool &&) = delete; linux_distributed_pool &operator=(linux_distributed_pool const &) = delete; - linux_distributed_pool(numa_topology_t topo = {}) noexcept - : linux_distributed_pool("fork_union", std::move(topo)) {} + linux_distributed_pool(numa_topology_t topo = {}) noexcept : linux_distributed_pool("forkunion", std::move(topo)) {} explicit linux_distributed_pool(char const *name, numa_topology_t topo = {}) noexcept : topology_(std::move(topo)) { // Accept null or empty names by falling back to a sensible default - char const *effective_name = (name && name[0] != '\0') ? name : "fork_union"; + char const *effective_name = (name && name[0] != '\0') ? name : "forkunion"; std::strncpy(name_, effective_name, sizeof(name_) - 1); name_[sizeof(name_) - 1] = '\0'; } @@ -4069,5 +4104,5 @@ struct log_capabilities_t { }; #pragma endregion - Logging -} // namespace fork_union +} // namespace forkunion } // namespace ashvardanian diff --git a/rust/lib.rs b/rust/forkunion.rs similarity index 78% rename from rust/lib.rs rename to rust/forkunion.rs index 4e83631..732b7d9 100644 --- a/rust/lib.rs +++ b/rust/forkunion.rs @@ -1,6 +1,6 @@ //! Low-latency OpenMP-style NUMA-aware cross-platform fine-grained parallelism library. //! -//! Fork Union provides a minimalistic cross-platform thread-pool implementation and Parallel Algorithms, +//! ForkUnion provides a minimalistic cross-platform thread-pool implementation and Parallel Algorithms, //! avoiding dynamic memory allocations, exceptions, system calls, and heavy Compare-And-Swap instructions. //! The library leverages the "weak memory model" to allow Arm and IBM Power CPUs to aggressively optimize //! execution at runtime. It also aggressively tests against overflows on smaller index types, and is safe @@ -27,7 +27,7 @@ use core::sync::atomic::{AtomicBool, Ordering}; /// Default alignment for preventing false sharing between threads. /// /// Set to 128 bytes to account for adjacent cache-line prefetching on modern CPUs. -/// This matches the C++ `default_alignment_k` constant defined in `fork_union.hpp`. +/// This matches the C++ `default_alignment_k` constant defined in `forkunion.hpp`. /// /// On x86, most CPUs fetch 2 cache lines (128 bytes) at once with spatial prefetching enabled. /// This conservative padding prevents false sharing even with aggressive prefetch settings. @@ -45,7 +45,7 @@ pub const DEFAULT_ALIGNMENT: usize = 128; /// # Examples /// /// ```rust -/// use fork_union::{CacheAligned, ThreadPool}; +/// use forkunion::{CacheAligned, ThreadPool}; /// /// let mut pool = ThreadPool::try_spawn(4).unwrap(); /// let data: Vec = (0..1000).collect(); @@ -66,6 +66,12 @@ pub const DEFAULT_ALIGNMENT: usize = 128; #[derive(Clone, Copy, Debug, Default)] pub struct CacheAligned(pub T); +// Compile-time assertion that alignment matches DEFAULT_ALIGNMENT +const _: () = assert!( + core::mem::align_of::>() == DEFAULT_ALIGNMENT, + "CacheAligned alignment must match DEFAULT_ALIGNMENT" +); + /// A generic spin mutex that uses CPU-specific pause instructions for efficient busy-waiting. /// /// This is a low-level synchronization primitive that spins on a busy loop rather than @@ -79,7 +85,7 @@ pub struct CacheAligned(pub T); /// # Examples /// /// ```rust -/// use fork_union::*; +/// use forkunion::*; /// /// // Create a spin mutex with pause instructions enabled /// let mutex = BasicSpinMutex::::new(42); @@ -111,7 +117,7 @@ impl BasicSpinMutex { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let mutex = BasicSpinMutex::::new(0); /// ``` @@ -130,7 +136,7 @@ impl BasicSpinMutex { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let mutex = BasicSpinMutex::::new(0); /// let mut guard = mutex.lock(); @@ -158,7 +164,7 @@ impl BasicSpinMutex { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let mutex = BasicSpinMutex::::new(0); /// @@ -189,7 +195,7 @@ impl BasicSpinMutex { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let mutex = BasicSpinMutex::::new(0); /// assert!(!mutex.is_locked()); @@ -213,7 +219,7 @@ impl BasicSpinMutex { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let mutex = BasicSpinMutex::::new(42); /// let data = mutex.into_inner(); @@ -231,7 +237,7 @@ impl BasicSpinMutex { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let mut mutex = BasicSpinMutex::::new(0); /// *mutex.get_mut() = 42; @@ -298,7 +304,7 @@ impl<'a, T, const PAUSE: bool> Drop for BasicSpinMutexGuard<'a, T, PAUSE> { /// # Examples /// /// ```rust -/// use fork_union::*; +/// use forkunion::*; /// /// let mutex = SpinMutex::new(42); /// let mut guard = mutex.lock(); @@ -500,17 +506,17 @@ pub fn numa_enabled() -> bool { unsafe { fu_enabled_numa() != 0 } } -/// Returns the major version number of the Fork Union library. +/// Returns the major version number of the ForkUnion library. pub fn version_major() -> usize { unsafe { fu_version_major() as usize } } -/// Returns the minor version number of the Fork Union library. +/// Returns the minor version number of the ForkUnion library. pub fn version_minor() -> usize { unsafe { fu_version_minor() as usize } } -/// Returns the patch version number of the Fork Union library. +/// Returns the patch version number of the ForkUnion library. pub fn version_patch() -> usize { unsafe { fu_version_patch() as usize } } @@ -544,7 +550,7 @@ pub fn version() -> (usize, usize, usize) { /// Basic usage with simple computations: /// /// ```rust -/// use fork_union::*; +/// use forkunion::*; /// /// // Create a thread pool with 4 threads /// let mut pool = spawn(4); @@ -626,7 +632,7 @@ impl ThreadPool { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// // Create a pool that uses 4 threads total (3 spawned + caller) /// let pool = ThreadPool::try_spawn(4).expect("Failed to create thread pool"); @@ -650,7 +656,7 @@ impl ThreadPool { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let pool = ThreadPool::try_named_spawn("worker_pool", 4).expect("Failed to create thread pool"); /// assert_eq!(pool.threads(), 4); @@ -684,7 +690,7 @@ impl ThreadPool { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let pool = spawn(8); /// let total_colocations = pool.colocations(); @@ -734,7 +740,7 @@ impl ThreadPool { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let mut pool = spawn(4); /// @@ -771,7 +777,7 @@ impl ThreadPool { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let mut pool = spawn(4); /// @@ -806,7 +812,7 @@ impl ThreadPool { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let mut pool = spawn(4); /// @@ -841,7 +847,7 @@ impl ThreadPool { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let mut pool = spawn(4); /// @@ -878,7 +884,7 @@ impl ThreadPool { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let mut pool = spawn(4); /// @@ -924,6 +930,9 @@ pub struct AllocationResult { allocated_bytes: usize, bytes_per_page: usize, numa_node: usize, + // For over-aligned allocations, tracks the unaligned pointer/size for freeing + overaligned_ptr: Option>, + overaligned_bytes: Option, } impl AllocationResult { @@ -988,11 +997,10 @@ impl AllocationResult { impl Drop for AllocationResult { fn drop(&mut self) { unsafe { - fu_free( - self.numa_node, - self.ptr.as_ptr() as *mut c_void, - self.allocated_bytes, - ); + // Use unaligned pointer/size if this was an over-aligned allocation + let ptr = self.overaligned_ptr.unwrap_or(self.ptr); + let bytes = self.overaligned_bytes.unwrap_or(self.allocated_bytes); + fu_free(self.numa_node, ptr.as_ptr() as *mut c_void, bytes); } } } @@ -1011,7 +1019,7 @@ unsafe impl Sync for AllocationResult {} /// # Examples /// /// ```rust -/// use fork_union::*; +/// use forkunion::*; /// let allocator = PinnedAllocator::new(0).expect("Failed to create alloc for NUMA node 0"); /// let allocation = allocator.allocate(1024).expect("Failed to allocate 1024 bytes"); /// @@ -1040,7 +1048,7 @@ impl PinnedAllocator { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// // Create allocator for the first NUMA node /// let allocator = PinnedAllocator::new(0).expect("NUMA node 0 should be available"); @@ -1091,7 +1099,7 @@ impl PinnedAllocator { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let allocator = PinnedAllocator::new(0).unwrap(); /// let allocation = allocator.allocate_at_least(1024).expect("Failed to allocate memory"); @@ -1131,6 +1139,8 @@ impl PinnedAllocator { allocated_bytes, bytes_per_page, numa_node: self.numa_node, + overaligned_ptr: None, + overaligned_bytes: None, }) } } @@ -1148,7 +1158,7 @@ impl PinnedAllocator { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let allocator = PinnedAllocator::new(0).unwrap(); /// let allocation = allocator.allocate(1024).expect("Failed to allocate memory"); @@ -1181,6 +1191,8 @@ impl PinnedAllocator { allocated_bytes: bytes, bytes_per_page: 0, // Not provided by fu_allocate numa_node: self.numa_node, + overaligned_ptr: None, + overaligned_bytes: None, }) } } @@ -1194,7 +1206,7 @@ impl PinnedAllocator { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let allocator = PinnedAllocator::new(0).unwrap(); /// let mut allocation = allocator.allocate_for::(100).expect("Failed to allocate"); @@ -1212,8 +1224,37 @@ impl PinnedAllocator { /// assert_eq!(slice[99], 12345); /// ``` pub fn allocate_for(&self, count: usize) -> Option { - let bytes = count.checked_mul(core::mem::size_of::())?; - self.allocate(bytes) + let size = core::mem::size_of::(); + let align = core::mem::align_of::(); + let bytes = count.checked_mul(size)?; + + // If alignment is <= default malloc alignment (16 bytes), use simple path + if align <= 16 { + return self.allocate(bytes); + } + + // For over-aligned types (like CacheAligned with 128-byte alignment), + // we need to over-allocate and manually align the pointer + let padding = align - 1; + let total_bytes = bytes.checked_add(padding)?; + + let mut allocation = self.allocate(total_bytes)?; + + // Save unaligned pointer and size for freeing + let unaligned_ptr = allocation.ptr; + let unaligned_bytes = allocation.allocated_bytes; + + // Calculate aligned pointer + let ptr = allocation.as_ptr() as usize; + let aligned_ptr = (ptr + padding) & !(align - 1); + + // Adjust the allocation to point to the aligned address + allocation.ptr = unsafe { core::ptr::NonNull::new_unchecked(aligned_ptr as *mut u8) }; + allocation.allocated_bytes = bytes; + allocation.overaligned_ptr = Some(unaligned_ptr); + allocation.overaligned_bytes = Some(unaligned_bytes); + + Some(allocation) } /// Allocates memory for at least the specified number of elements of type T. @@ -1227,7 +1268,7 @@ impl PinnedAllocator { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let allocator = PinnedAllocator::new(0).unwrap(); /// let mut allocation = allocator.allocate_for_at_least::(1000).expect("Failed to allocate"); @@ -1261,7 +1302,7 @@ impl PinnedAllocator { /// # Examples /// /// ```rust -/// use fork_union::*; +/// use forkunion::*; /// /// let allocator = default_numa_allocator().expect("No NUMA nodes available"); /// let allocation = allocator.allocate(1024).expect("Failed to allocate"); @@ -1292,7 +1333,7 @@ pub fn default_numa_allocator() -> Option { /// # Examples /// /// ```rust -/// use fork_union::*; +/// use forkunion::*; /// /// // Create a vector on NUMA node 0 /// let allocator = PinnedAllocator::new(0).expect("Failed to create alloc"); @@ -1331,7 +1372,7 @@ impl PinnedVec { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let allocator = PinnedAllocator::new(0).expect("Failed to create alloc"); /// let vec = PinnedVec::::new_in(allocator); @@ -1362,7 +1403,7 @@ impl PinnedVec { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let allocator = PinnedAllocator::new(0).expect("Failed to create alloc"); /// let vec = PinnedVec::::with_capacity_in(allocator, 100).expect("Failed to create vec"); @@ -1418,7 +1459,7 @@ impl PinnedVec { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let allocator = PinnedAllocator::new(0).expect("Failed to create alloc"); /// let mut vec = PinnedVec::::new_in(allocator); @@ -1476,7 +1517,7 @@ impl PinnedVec { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let allocator = PinnedAllocator::new(0).expect("Failed to create alloc"); /// let mut vec = PinnedVec::::new_in(allocator); @@ -1506,7 +1547,7 @@ impl PinnedVec { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let allocator = PinnedAllocator::new(0).expect("Failed to create alloc"); /// let mut vec = PinnedVec::::new_in(allocator); @@ -1531,7 +1572,7 @@ impl PinnedVec { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let allocator = PinnedAllocator::new(0).expect("Failed to create alloc"); /// let mut vec = PinnedVec::::new_in(allocator); @@ -1803,7 +1844,7 @@ impl PinnedVec { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let allocator = PinnedAllocator::new(0).expect("Failed to create alloc"); /// let mut vec = PinnedVec::::with_capacity_in(allocator, 5).expect("Failed to create vec"); @@ -1827,7 +1868,7 @@ impl PinnedVec { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let allocator = PinnedAllocator::new(0).expect("Failed to create alloc"); /// let mut vec = PinnedVec::::with_capacity_in(allocator, 5).expect("Failed to create vec"); @@ -1874,7 +1915,7 @@ unsafe impl Sync for PinnedVec {} /// # Examples /// /// ```rust -/// use fork_union::*; +/// use forkunion::*; /// /// let mut pool = ThreadPool::try_spawn(4).expect("Failed to create pool"); /// let mut rr_vec = RoundRobinVec::::new().expect("Failed to create RoundRobinVec"); @@ -1898,7 +1939,7 @@ impl RoundRobinVec { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let rr_vec = RoundRobinVec::::new().expect("Failed to create RoundRobinVec"); /// assert_eq!(rr_vec.colocations_count(), count_colocations()); @@ -1945,7 +1986,7 @@ impl RoundRobinVec { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let rr_vec = RoundRobinVec::::with_capacity_per_colocation(1000) /// .expect("Failed to create RoundRobinVec"); @@ -2070,7 +2111,7 @@ impl RoundRobinVec { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let mut rr_vec = RoundRobinVec::::new().expect("Failed to create RoundRobinVec"); /// // Add some elements... @@ -2129,7 +2170,7 @@ impl RoundRobinVec { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let mut rr_vec = RoundRobinVec::::new().expect("Failed to create RoundRobinVec"); /// rr_vec.push(42).expect("Failed to push"); @@ -2198,7 +2239,7 @@ impl RoundRobinVec { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let mut rr_vec = RoundRobinVec::::new().expect("Failed to create RoundRobinVec"); /// rr_vec.push(42).expect("Failed to push"); @@ -2267,7 +2308,7 @@ impl RoundRobinVec { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let mut pool = ThreadPool::try_spawn(4).expect("Failed to create pool"); /// let mut rr_vec = RoundRobinVec::::with_capacity_per_colocation(1000) @@ -2290,21 +2331,22 @@ impl RoundRobinVec { let pool_ptr = SafePtr(pool as *const ThreadPool as *mut ThreadPool); pool.for_threads(move |thread_index, colocation_index| { - if colocation_index < colocations_count { - // Get the specific pinned vector for this NUMA node - let node_vec = safe_ptr.get_mut_at(colocation_index); - let pool = pool_ptr.get_mut(); - - let threads_in_colocation = pool.count_threads_in(colocation_index); - let thread_local_index = pool.locate_thread_in(thread_index, colocation_index); - let split = IndexedSplit::new(node_vec.len(), threads_in_colocation); - let range = split.get(thread_local_index); + if colocation_index >= colocations_count { + return; + } - // Fill the assigned range of this thread - for idx in range { - if let Some(element) = node_vec.get_mut(idx) { - *element = value.clone(); - } + let node_vec = safe_ptr.get_mut_at(colocation_index); + let pool = pool_ptr.get_mut(); + + let threads_in_colocation = pool.count_threads_in(colocation_index); + let thread_local_index = pool.locate_thread_in(thread_index, colocation_index); + let split = IndexedSplit::new(node_vec.len(), threads_in_colocation); + let range = split.get(thread_local_index); + + // Fill the assigned range of this thread + for idx in range { + if let Some(element) = node_vec.get_mut(idx) { + *element = value.clone(); } } }); @@ -2321,7 +2363,7 @@ impl RoundRobinVec { /// # Examples /// /// ```rust - /// use fork_union::*; + /// use forkunion::*; /// /// let mut pool = ThreadPool::try_spawn(4).expect("Failed to create pool"); /// let mut rr_vec = RoundRobinVec::::with_capacity_per_colocation(1000) @@ -2346,22 +2388,23 @@ impl RoundRobinVec { let pool_ptr = SafePtr(pool as *const ThreadPool as *mut ThreadPool); pool.for_threads(move |thread_index, colocation_index| { - if colocation_index < colocations_count { - // Get the specific pinned vector for this NUMA node - let node_vec = safe_ptr.get_mut_at(colocation_index); - let f_ref = f_ptr.get_mut(); - let pool = pool_ptr.get_mut(); - - let threads_in_colocation = pool.count_threads_in(colocation_index); - let thread_local_index = pool.locate_thread_in(thread_index, colocation_index); - let split = IndexedSplit::new(node_vec.len(), threads_in_colocation); - let range = split.get(thread_local_index); + if colocation_index >= colocations_count { + return; + } - // Fill the assigned range of this thread - for idx in range { - if let Some(element) = node_vec.get_mut(idx) { - *element = f_ref(); - } + let node_vec = safe_ptr.get_mut_at(colocation_index); + let f_ref = f_ptr.get_mut(); + let pool = pool_ptr.get_mut(); + + let threads_in_colocation = pool.count_threads_in(colocation_index); + let thread_local_index = pool.locate_thread_in(thread_index, colocation_index); + let split = IndexedSplit::new(node_vec.len(), threads_in_colocation); + let range = split.get(thread_local_index); + + // Fill the assigned range of this thread + for idx in range { + if let Some(element) = node_vec.get_mut(idx) { + *element = f_ref(); } } }); @@ -2378,22 +2421,23 @@ impl RoundRobinVec { let pool_ptr = SafePtr(pool as *const ThreadPool as *mut ThreadPool); pool.for_threads(move |thread_index, colocation_index| { - if colocation_index < colocations_count { - // Get the specific pinned vector for this NUMA node - let node_vec = safe_ptr.get_mut_at(colocation_index); - let pool = pool_ptr.get_mut(); - - let threads_in_colocation = pool.count_threads_in(colocation_index); - let thread_local_index = pool.locate_thread_in(thread_index, colocation_index); - let split = IndexedSplit::new(node_vec.len(), threads_in_colocation); - let range = split.get(thread_local_index); + if colocation_index >= colocations_count { + return; + } - // Drop elements in the assigned range - unsafe { - let ptr = node_vec.as_mut_ptr(); - for idx in range { - core::ptr::drop_in_place(ptr.add(idx)); - } + let node_vec = safe_ptr.get_mut_at(colocation_index); + let pool = pool_ptr.get_mut(); + + let threads_in_colocation = pool.count_threads_in(colocation_index); + let thread_local_index = pool.locate_thread_in(thread_index, colocation_index); + let split = IndexedSplit::new(node_vec.len(), threads_in_colocation); + let range = split.get(thread_local_index); + + // Drop elements in the assigned range + unsafe { + let ptr = node_vec.as_mut_ptr(); + for idx in range { + core::ptr::drop_in_place(ptr.add(idx)); } } }); @@ -2435,18 +2479,21 @@ impl RoundRobinVec { let elements_per_node = new_len / colocations_count; let extra_elements = new_len % colocations_count; - // Step 1: Centrally handle reallocation for each NUMA node - for i in 0..colocations_count { - let node_len = if i < extra_elements { + // Helper to calculate target length for a colocation + let node_len = |col_idx: usize| -> usize { + if col_idx < extra_elements { elements_per_node + 1 } else { elements_per_node - }; + } + }; + // Step 1: Centrally handle reallocation for each NUMA node + for i in 0..colocations_count { + let target_len = node_len(i); let current_len = self.colocations[i].len(); - if node_len > current_len { - // Need to reserve more capacity - self.colocations[i].reserve(node_len - current_len)?; + if target_len > current_len { + self.colocations[i].reserve(target_len - current_len)?; } } @@ -2455,52 +2502,43 @@ impl RoundRobinVec { let pool_ptr = SafePtr(pool as *const ThreadPool as *mut ThreadPool); pool.for_threads(move |thread_index, colocation_index| { - if colocation_index < colocations_count { - // Get the specific pinned vector for this NUMA node - let node_vec = safe_ptr.get_mut_at(colocation_index); - let pool = pool_ptr.get_mut(); + if colocation_index >= colocations_count { + return; + } - let node_len = if colocation_index < extra_elements { - elements_per_node + 1 - } else { - elements_per_node - }; - - let current_len = node_vec.len(); - let threads_in_colocation = pool.count_threads_in(colocation_index); - let thread_local_index = pool.locate_thread_in(thread_index, colocation_index); - - match node_len.cmp(¤t_len) { - std::cmp::Ordering::Greater => { - // Growing: construct new elements in parallel - let new_elements = node_len - current_len; - let split = IndexedSplit::new(new_elements, threads_in_colocation); - let range = split.get(thread_local_index); - - unsafe { - let ptr = node_vec.as_mut_ptr(); - for i in range { - let idx = current_len + i; - core::ptr::write(ptr.add(idx), value.clone()); - } - } - } - std::cmp::Ordering::Less => { - // Shrinking: drop elements in parallel - let elements_to_drop = current_len - node_len; - let split = IndexedSplit::new(elements_to_drop, threads_in_colocation); - let range = split.get(thread_local_index); - - unsafe { - let ptr = node_vec.as_mut_ptr(); - for i in range { - let idx = node_len + i; - core::ptr::drop_in_place(ptr.add(idx)); - } - } + let node_vec = safe_ptr.get_mut_at(colocation_index); + let pool = pool_ptr.get_mut(); + let target_len = node_len(colocation_index); + let current_len = node_vec.len(); + if target_len == current_len { + return; + } + + let threads_in_colocation = pool.count_threads_in(colocation_index); + let thread_local_index = pool.locate_thread_in(thread_index, colocation_index); + + if target_len > current_len { + // Growing: construct new elements in parallel + let new_elements = target_len - current_len; + let split = IndexedSplit::new(new_elements, threads_in_colocation); + let range = split.get(thread_local_index); + + unsafe { + let ptr = node_vec.as_mut_ptr(); + for i in range { + core::ptr::write(ptr.add(current_len + i), value.clone()); } - std::cmp::Ordering::Equal => { - // No change needed + } + } else { + // Shrinking: drop elements in parallel + let elements_to_drop = current_len - target_len; + let split = IndexedSplit::new(elements_to_drop, threads_in_colocation); + let range = split.get(thread_local_index); + + unsafe { + let ptr = node_vec.as_mut_ptr(); + for i in range { + core::ptr::drop_in_place(ptr.add(target_len + i)); } } } @@ -2508,16 +2546,11 @@ impl RoundRobinVec { // Step 3: Update lengths after parallel operations for i in 0..colocations_count { - let node_len = if i < extra_elements { - elements_per_node + 1 - } else { - elements_per_node - }; - self.colocations[i].len = node_len; + self.colocations[i].len = node_len(i); } self.total_length = new_len; - self.total_capacity = self.capacity(); // Recalculate total capacity + self.total_capacity = self.capacity(); Ok(()) } } @@ -2571,7 +2604,7 @@ unsafe impl Sync for RoundRobinVec {} /// # Examples /// /// ```rust -/// use fork_union::*; +/// use forkunion::*; /// /// let data = vec![1, 2, 3, 4, 5]; /// let sync_ptr = SyncConstPtr::new(data.as_ptr()); @@ -2662,6 +2695,37 @@ impl SyncMutPtr { unsafe impl Send for SyncMutPtr {} unsafe impl Sync for SyncMutPtr {} +/// Sync wrapper for single-write cells used in early-exit operations. +/// +/// # Safety +/// +/// This is safe because: +/// - Only one thread writes (enforced by AtomicBool in caller) +/// - Write happens-before any subsequent read (synchronized by atomic operations) +/// - Final read happens after all threads finish (enforced by drive() completion) +struct SyncOnceCell { + inner: UnsafeCell>, +} + +unsafe impl Sync for SyncOnceCell {} + +impl SyncOnceCell { + const fn new() -> Self { + Self { + inner: UnsafeCell::new(None), + } + } + + /// SAFETY: Caller must ensure only one thread calls this + unsafe fn set(&self, value: T) { + *self.inner.get() = Some(value); + } + + fn into_inner(self) -> Option { + self.inner.into_inner() + } +} + /// Scheduler that uses static chunk assignment. #[derive(Clone, Copy, Debug)] pub struct StaticScheduler; @@ -2866,6 +2930,451 @@ where fold_with_scratch(pool, iterator, schedule, scratch, fold); } + /// Parallel reduction with caller-provided scratch buffer. + /// + /// Reduces items in parallel by folding into per-thread accumulators, + /// then combining results on the caller thread. Uses cache-aligned scratch + /// to prevent false sharing. Indexes by thread_index (works with dynamic scheduling). + /// + /// # Arguments + /// * `scratch` - Per-thread accumulators (must be `>= pool.threads()`) + /// * `fold` - Function to accumulate items: `fn(&mut T, I::Item, Prong)` + /// * `combine` - Function to merge two accumulators: `fn(T, T) -> T` + /// + /// # Returns + /// The final reduced value of type `T` + /// + /// # Example + /// ``` + /// use forkunion::*; + /// let mut pool = ThreadPool::try_spawn(4).unwrap(); + /// let data: Vec = (0..1000).collect(); + /// let mut scratch: Vec> = + /// (0..pool.threads()).map(|_| CacheAligned(0)).collect(); + /// + /// let total = (&data[..]).into_par_iter().with_pool(&mut pool) + /// .reduce_with_scratch( + /// scratch.as_mut_slice(), + /// |acc, value, _| acc.0 += *value, + /// |a, b| a.0 += b.0, + /// ); + /// ``` + pub fn reduce_with_scratch(self, scratch: &mut [T], fold: F, combine: C) -> T + where + T: Send + Default, + F: Fn(&mut T, I::Item, Prong) + Sync, + C: Fn(&mut T, T), + { + let ParallelRunner { + pool, + iterator, + schedule, + } = self; + + // Fold phase: accumulate into per-thread slots + fold_with_scratch(pool, iterator, schedule, scratch, fold); + + // Combine phase: merge all slots into first slot in-place + let (first, rest) = scratch + .split_first_mut() + .expect("scratch must not be empty"); + for slot in rest { + let value = core::mem::take(slot); + combine(first, value); + } + core::mem::take(first) + } + + /// Executes a fallible operation on each item, stopping at the first error. + /// + /// Uses cooperative cancellation: once an error occurs, no further items are processed. + /// Items already "in flight" may still complete, but new items won't start processing. + /// + /// # Returns + /// + /// - `Ok(())` if all items were processed successfully or were skipped after stop + /// - `Err(E)` with the first error encountered + /// + /// # Performance + /// + /// Overhead is one atomic load per item (~2% in compute-bound workloads). + /// The atomic swap on error is negligible as it happens at most once. + /// + /// # Example + /// + /// ``` + /// use forkunion::*; + /// + /// fn validate(x: &u64) -> Result<(), &'static str> { + /// if *x < 100 { Ok(()) } else { Err("value too large") } + /// } + /// + /// let mut pool = ThreadPool::try_spawn(4).unwrap(); + /// let data: Vec = (0..50).collect(); + /// + /// let result = (&data[..]) + /// .into_par_iter() + /// .with_pool(&mut pool) + /// .try_for_each(|x, _| validate(x)); + /// + /// assert!(result.is_ok()); + /// ``` + pub fn try_for_each(self, function: F) -> Result<(), E> + where + F: Fn(I::Item, Prong) -> Result<(), E> + Sync, + E: Send, + { + use core::sync::atomic::{AtomicBool, Ordering}; + + let ParallelRunner { + pool, + iterator, + schedule, + } = self; + + let stop = AtomicBool::new(false); + let first_err = SyncOnceCell::new(); + let f_ptr = SyncConstPtr::new(&function as *const F); + + iterator.drive(pool, schedule, &|item, prong| { + // Check if we should stop (Acquire: see all writes before Release swap) + if stop.load(Ordering::Acquire) { + return; + } + + let func = unsafe { &*f_ptr.as_ptr() }; + if let Err(e) = func(item, prong) { + // Try to set stop flag (Release: make error write visible to Acquire loads) + let already_stopped = stop.swap(true, Ordering::Release); + if !already_stopped { + // SAFETY: Only one thread sets stop to true, so only one write + unsafe { first_err.set(e) }; + } + } + }); + + // SAFETY: All worker threads finished, exclusive access + match first_err.into_inner() { + Some(e) => Err(e), + None => Ok(()), + } + } + + /// Searches for any element that matches a predicate (non-deterministic). + /// + /// Uses cooperative cancellation: once a match is found, no further items are processed. + /// If multiple items match, any one of them may be returned. + /// + /// # Returns + /// + /// - `Some(item)` if a matching item was found + /// - `None` if no item matched or the iterator was empty + /// + /// # Example + /// + /// ``` + /// use forkunion::*; + /// + /// let mut pool = ThreadPool::try_spawn(4).unwrap(); + /// let data: Vec = (0..1000).collect(); + /// + /// let found = (&data[..]) + /// .into_par_iter() + /// .with_pool(&mut pool) + /// .find_any(|&&x| x == 42); + /// + /// assert_eq!(found, Some(&42)); + /// ``` + pub fn find_any

(self, predicate: P) -> Option + where + I::Item: Send, + P: Fn(&I::Item) -> bool + Sync, + { + use core::sync::atomic::{AtomicBool, Ordering}; + + let ParallelRunner { + pool, + iterator, + schedule, + } = self; + + let stop = AtomicBool::new(false); + let found = SyncOnceCell::new(); + let p_ptr = SyncConstPtr::new(&predicate as *const P); + + iterator.drive(pool, schedule, &|item, _prong| { + // Check if already found (Acquire: see all writes before Release swap) + if stop.load(Ordering::Acquire) { + return; + } + + let pred = unsafe { &*p_ptr.as_ptr() }; + if pred(&item) { + // Try to set stop flag (Release: make item write visible to Acquire loads) + let already_stopped = stop.swap(true, Ordering::Release); + if !already_stopped { + // SAFETY: Only one thread sets stop to true, so only one write + unsafe { found.set(item) }; + } + } + }); + + // SAFETY: All worker threads finished, exclusive access + found.into_inner() + } + + /// Searches for the first element that matches a predicate (deterministic, by index). + /// + /// Returns the element with the smallest `task_index` among all matches. + /// Uses `fetch_min` to track the minimum index found so far. + /// + /// # Returns + /// + /// - `Some(item)` with the lowest index if any match was found + /// - `None` if no item matched or the iterator was empty + /// + /// # Example + /// + /// ``` + /// use forkunion::*; + /// + /// let mut pool = ThreadPool::try_spawn(4).unwrap(); + /// let data: Vec = vec![10, 20, 30, 20, 10]; + /// + /// let found = (&data[..]) + /// .into_par_iter() + /// .with_pool(&mut pool) + /// .find_first(|&&x| x == 20); + /// + /// assert_eq!(found, Some(&20)); // Index 1, not 3 + /// ``` + pub fn find_first

(self, predicate: P) -> Option + where + I::Item: Send, + P: Fn(&I::Item) -> bool + Sync, + { + use core::sync::atomic::{AtomicUsize, Ordering}; + + let ParallelRunner { + pool, + iterator, + schedule, + } = self; + + let min_index = AtomicUsize::new(usize::MAX); + let found = BasicSpinMutex::<_, true>::new(None); + let p_ptr = SyncConstPtr::new(&predicate as *const P); + + iterator.drive(pool, schedule, &|item, prong| { + let pred = unsafe { &*p_ptr.as_ptr() }; + if pred(&item) { + let my_index = prong.task_index; + let old_min = min_index.fetch_min(my_index, Ordering::Relaxed); + if my_index < old_min { + // We have a new minimum, update the stored item + *found.lock() = Some(item); + } + } + }); + + found.into_inner() + } + + /// Searches for the last element that matches a predicate (deterministic, by index). + /// + /// Returns the element with the largest `task_index` among all matches. + /// Uses `fetch_max` to track the maximum index found so far. + /// + /// # Returns + /// + /// - `Some(item)` with the highest index if any match was found + /// - `None` if no item matched or the iterator was empty + /// + /// # Example + /// + /// ``` + /// use forkunion::*; + /// + /// let mut pool = ThreadPool::try_spawn(4).unwrap(); + /// let data: Vec = vec![10, 20, 30, 20, 10]; + /// + /// let found = (&data[..]) + /// .into_par_iter() + /// .with_pool(&mut pool) + /// .find_last(|&&x| x == 20); + /// + /// assert_eq!(found, Some(&20)); // Index 3, not 1 + /// ``` + pub fn find_last

(self, predicate: P) -> Option + where + I::Item: Send, + P: Fn(&I::Item) -> bool + Sync, + { + use core::sync::atomic::{AtomicUsize, Ordering}; + + let ParallelRunner { + pool, + iterator, + schedule, + } = self; + + let max_index = AtomicUsize::new(0); + let found = BasicSpinMutex::<_, true>::new(None); + let p_ptr = SyncConstPtr::new(&predicate as *const P); + + iterator.drive(pool, schedule, &|item, prong| { + let pred = unsafe { &*p_ptr.as_ptr() }; + if pred(&item) { + let my_index = prong.task_index; + let old_max = max_index.fetch_max(my_index, Ordering::Relaxed); + if my_index > old_max { + // We have a new maximum, update the stored item + *found.lock() = Some(item); + } + } + }); + + found.into_inner() + } + + /// Returns `true` if any item matches the predicate. + /// + /// Stops searching after the first match is found. + /// + /// # Example + /// + /// ``` + /// use forkunion::*; + /// + /// let mut pool = ThreadPool::try_spawn(4).unwrap(); + /// let data: Vec = (0..1000).collect(); + /// + /// let has_large = (&data[..]) + /// .into_par_iter() + /// .with_pool(&mut pool) + /// .any(|&&x| x > 500); + /// + /// assert!(has_large); + /// ``` + pub fn any

(self, predicate: P) -> bool + where + I::Item: Send, + P: Fn(&I::Item) -> bool + Sync, + { + self.find_any(predicate).is_some() + } + + /// Returns `true` if all items match the predicate. + /// + /// Stops searching after the first non-match is found. + /// + /// # Example + /// + /// ``` + /// use forkunion::*; + /// + /// let mut pool = ThreadPool::try_spawn(4).unwrap(); + /// let data: Vec = (0..100).collect(); + /// + /// let all_small = (&data[..]) + /// .into_par_iter() + /// .with_pool(&mut pool) + /// .all(|&&x| x < 200); + /// + /// assert!(all_small); + /// ``` + pub fn all

(self, predicate: P) -> bool + where + I::Item: Send, + P: Fn(&I::Item) -> bool + Sync, + { + !self.any(|x| !predicate(x)) + } + + /// Fold with early-exit on error, using caller-provided scratch buffer. + /// + /// Similar to `fold_with_scratch`, but allows the fold function to return `Result`. + /// Stops processing on the first error. Scratch buffers are indexed by `thread_index`. + /// + /// # Arguments + /// + /// * `scratch` - Per-thread accumulators (must be `>= pool.threads()`) + /// * `fold` - Fallible fold function: `fn(&mut T, I::Item, Prong) -> Result<(), E>` + /// + /// # Returns + /// + /// - `Ok(())` if all items were folded successfully + /// - `Err(E)` with the first error encountered + /// + /// # Example + /// + /// ``` + /// use forkunion::*; + /// + /// fn checked_add(acc: &mut u64, value: &u64) -> Result<(), &'static str> { + /// *acc = acc.checked_add(*value).ok_or("overflow")?; + /// Ok(()) + /// } + /// + /// let mut pool = ThreadPool::try_spawn(4).unwrap(); + /// let data: Vec = (1..100).collect(); + /// let mut scratch: Vec> = + /// (0..pool.threads()).map(|_| CacheAligned(0)).collect(); + /// + /// let result = (&data[..]) + /// .into_par_iter() + /// .with_pool(&mut pool) + /// .try_fold_with_scratch(scratch.as_mut_slice(), |acc, value, _| { + /// checked_add(&mut acc.0, value) + /// }); + /// + /// assert!(result.is_ok()); + /// ``` + pub fn try_fold_with_scratch(self, scratch: &mut [T], fold: F) -> Result<(), E> + where + T: Send, + F: Fn(&mut T, I::Item, Prong) -> Result<(), E> + Sync, + E: Send, + { + use core::sync::atomic::{AtomicBool, Ordering}; + + let ParallelRunner { + pool, + iterator, + schedule, + } = self; + + let stop = AtomicBool::new(false); + let first_err = SyncOnceCell::new(); + let f_ptr = SyncConstPtr::new(&fold as *const F); + let s_ptr = SyncMutPtr::new(scratch.as_mut_ptr()); + + iterator.drive(pool, schedule, &|item, prong| { + // Check if we should stop (Acquire: see all writes before Release swap) + if stop.load(Ordering::Acquire) { + return; + } + + let slot = unsafe { &mut *s_ptr.get(prong.thread_index) }; + let func = unsafe { &*f_ptr.as_ptr() }; + + if let Err(e) = func(slot, item, prong) { + // Try to set stop flag (Release: make error write visible to Acquire loads) + let already_stopped = stop.swap(true, Ordering::Release); + if !already_stopped { + // SAFETY: Only one thread sets stop to true, so only one write + unsafe { first_err.set(e) }; + } + } + }); + + // SAFETY: All worker threads finished, exclusive access + match first_err.into_inner() { + Some(e) => Err(e), + None => Ok(()), + } + } + pub fn with_schedule(self, schedule: S2) -> ParallelRunner<'pool, I, S2> where S2: ParallelSchedule, @@ -2879,6 +3388,113 @@ where } } +// Convenience methods using NUMA-aware RoundRobinVec for scratch buffers +// Each colocation gets its own CacheAligned accumulator pinned to local NUMA node! +impl<'pool, I, S> ParallelRunner<'pool, I, S> +where + I: ParallelIterator, + S: ParallelSchedule, +{ + /// Parallel reduction with NUMA-aware scratch allocation. + /// + /// Automatically allocates cache-aligned scratch buffers on each NUMA node + /// using `RoundRobinVec`. Each colocation gets one `CacheAligned` accumulator + /// pinned to its local memory - threads access local NUMA memory! + /// + /// Nearly identical to Rayon's reduce API, just requires explicit pool. + /// + /// # Arguments + /// * `init` - Function to create initial accumulator value + /// * `fold` - Function to accumulate items: `fn(&mut T, I::Item, Prong)` + /// * `combine` - Function to merge two accumulators: `fn(T, T) -> T` + /// + /// # Example + /// ``` + /// use forkunion::*; + /// let mut pool = ThreadPool::try_spawn(4).unwrap(); + /// let data: Vec = (0..1000).collect(); + /// + /// let total = (&data[..]).into_par_iter().with_pool(&mut pool) + /// .reduce(|| 0, |acc, value, _| *acc += *value, |a, b| a + b); + /// ``` + pub fn reduce(self, init: Init, fold: F, combine: C) -> T + where + Init: Fn() -> T + Sync, + T: Send + Sync + Default, + F: Fn(&mut T, I::Item, Prong) + Sync, + C: Fn(T, T) -> T, + { + // Handle empty iterators early + if self.iterator.is_empty() { + return init(); + } + + let threads = self.pool.threads(); + + // Create cache-aligned scratch: one CacheAligned per thread + // Note: Using PinnedVec per colocation for true NUMA-awareness would be ideal, + // but for simplicity we use a contiguous allocation here. The OS will still + // tend to place this on the NUMA node of the allocating thread. + let mut scratch = PinnedVec::with_capacity_in( + PinnedAllocator::new(0).expect("failed to get allocator"), + threads, + ) + .expect("failed to allocate scratch"); + + for _ in 0..threads { + scratch.push(CacheAligned(init())).expect("failed to push"); + } + + // Fold phase uses reduce_with_scratch which indexes by thread_index + self.reduce_with_scratch( + scratch.as_mut_slice(), + |acc, item, prong| fold(&mut acc.0, item, prong), + |a, b| { + let old_a = core::mem::take(&mut a.0); + a.0 = combine(old_a, b.0); + }, + ) + .0 + } + + /// Sum all items in parallel with NUMA-aware local accumulators. + /// + /// Works for owned values (usize, u64, etc.) and references (&u64, etc.). + /// + /// # Example + /// ``` + /// use forkunion::*; + /// let mut pool = ThreadPool::try_spawn(4).unwrap(); + /// let data = vec![1u64, 2, 3, 4, 5]; + /// let sum: u64 = (&data[..]).into_par_iter().with_pool(&mut pool).sum(); + /// assert_eq!(sum, 15); + /// ``` + pub fn sum(self) -> T + where + T: Send + + Sync + + Default + + Copy + + core::ops::AddAssign + + core::ops::Add, + { + self.reduce(T::default, |acc, item, _| *acc += item, |a, b| a + b) + } + + /// Count all items in parallel with NUMA-aware local counters. + /// + /// # Example + /// ``` + /// use forkunion::*; + /// let mut pool = ThreadPool::try_spawn(4).unwrap(); + /// let data: Vec = (0..1000).collect(); + /// let count = (&data[..]).into_par_iter().with_pool(&mut pool).count(); + /// ``` + pub fn count(self) -> usize { + self.reduce(|| 0usize, |acc, _item, _| *acc += 1, |a, b| a + b) + } +} + pub trait IntoParallelIterator { type Item; type Iter: ParallelIterator; @@ -4337,4 +4953,316 @@ mod tests { fn indexed_split_zero_threads() { IndexedSplit::new(10, 0); } + + #[test] + fn reduce_with_scratch_sum() { + let mut pool = spawn(hw_threads()); + let data: Vec = (0..1024).collect(); + let mut scratch: Vec> = + (0..pool.threads()).map(|_| CacheAligned(0)).collect(); + + let total = (&data[..]) + .into_par_iter() + .with_pool(&mut pool) + .reduce_with_scratch( + scratch.as_mut_slice(), + |acc, value, _| acc.0 += *value, + |a, b| a.0 += b.0, + ); + + assert_eq!(total.0, data.iter().sum()); + } + + #[test] + fn reduce_with_scratch_dynamic() { + let mut pool = spawn(hw_threads()); + let data: Vec = (0..1000).collect(); + let mut scratch: Vec> = + (0..pool.threads()).map(|_| CacheAligned(0)).collect(); + + let total = (&data[..]) + .into_par_iter() + .with_schedule(&mut pool, DynamicScheduler) + .reduce_with_scratch( + scratch.as_mut_slice(), + |a, v, _| a.0 += *v, + |x, y| x.0 += y.0, + ); + + assert_eq!(total.0, data.iter().sum()); + } + + #[test] + fn reduce_sum() { + let mut pool = spawn(hw_threads()); + let data: Vec = (1..=1000).collect(); + let total: u64 = (&data[..]).into_par_iter().with_pool(&mut pool).sum(); + assert_eq!(total, data.iter().sum()); + } + + #[test] + fn reduce_count() { + let mut pool = spawn(hw_threads()); + let data: Vec = (0..1000).collect(); + let count = (&data[..]).into_par_iter().with_pool(&mut pool).count(); + assert_eq!(count, 1000); + } + + #[test] + fn reduce_product() { + let mut pool = spawn(hw_threads()); + let data = vec![2u64, 3, 5, 7]; + let product = (&data[..]).into_par_iter().with_pool(&mut pool).reduce( + || 1u64, + |a, v, _| *a *= *v, + |x, y| x * y, + ); + assert_eq!(product, data.iter().product()); + } + + #[test] + fn reduce_empty() { + let mut pool = spawn(hw_threads()); + let data: Vec = vec![]; + assert_eq!( + (&data[..]) + .into_par_iter() + .with_pool(&mut pool) + .sum::(), + 0 + ); + } + + #[test] + fn reduce_range() { + let mut pool = spawn(hw_threads()); + let total: usize = (0..10_000).into_par_iter().with_pool(&mut pool).sum(); + assert_eq!(total, (0..10_000).sum()); + } + + // Early-exit API tests + + #[test] + fn try_for_each_success() { + let mut pool = spawn(hw_threads()); + let data: Vec = (0..1000).collect(); + let result = (&data[..]) + .into_par_iter() + .with_pool(&mut pool) + .try_for_each(|&x, _| if x < 1000 { Ok(()) } else { Err("too large") }); + assert!(result.is_ok()); + } + + #[test] + fn try_for_each_early_exit() { + let mut pool = spawn(hw_threads()); + let data: Vec = (0..1000).collect(); + let result = (&data[..]) + .into_par_iter() + .with_pool(&mut pool) + .try_for_each(|&x, _| if x < 500 { Ok(()) } else { Err(x) }); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(err >= 500 && err < 1000); + } + + #[test] + fn try_for_each_empty() { + let mut pool = spawn(hw_threads()); + let data: Vec = vec![]; + let result = (&data[..]) + .into_par_iter() + .with_pool(&mut pool) + .try_for_each(|&_x, _| -> Result<(), &str> { Err("should not run") }); + assert!(result.is_ok()); + } + + #[test] + fn find_any_found() { + let mut pool = spawn(hw_threads()); + let data: Vec = (0..1000).collect(); + let found = (&data[..]) + .into_par_iter() + .with_pool(&mut pool) + .find_any(|&&x| x == 42); + assert_eq!(found, Some(&42)); + } + + #[test] + fn find_any_not_found() { + let mut pool = spawn(hw_threads()); + let data: Vec = (0..1000).collect(); + let found = (&data[..]) + .into_par_iter() + .with_pool(&mut pool) + .find_any(|&&x| x == 2000); + assert_eq!(found, None); + } + + #[test] + fn find_any_empty() { + let mut pool = spawn(hw_threads()); + let data: Vec = vec![]; + let found = (&data[..]) + .into_par_iter() + .with_pool(&mut pool) + .find_any(|&&_x| true); + assert_eq!(found, None); + } + + #[test] + fn find_first_deterministic() { + let mut pool = spawn(hw_threads()); + let data: Vec = (0..1000).collect(); + // Find first even number >= 100 + let found = (&data[..]) + .into_par_iter() + .with_pool(&mut pool) + .find_first(|&&x| x >= 100 && x % 2 == 0); + assert_eq!(found, Some(&100)); + } + + #[test] + fn find_first_not_found() { + let mut pool = spawn(hw_threads()); + let data: Vec = (0..100).collect(); + let found = (&data[..]) + .into_par_iter() + .with_pool(&mut pool) + .find_first(|&&x| x >= 200); + assert_eq!(found, None); + } + + #[test] + fn find_last_deterministic() { + let mut pool = spawn(hw_threads()); + let data: Vec = (0..1000).collect(); + // Find last even number < 900 + let found = (&data[..]) + .into_par_iter() + .with_pool(&mut pool) + .find_last(|&&x| x < 900 && x % 2 == 0); + assert_eq!(found, Some(&898)); + } + + #[test] + fn find_last_not_found() { + let mut pool = spawn(hw_threads()); + let data: Vec = (0..100).collect(); + let found = (&data[..]) + .into_par_iter() + .with_pool(&mut pool) + .find_last(|&&x| x >= 200); + assert_eq!(found, None); + } + + #[test] + fn any_true() { + let mut pool = spawn(hw_threads()); + let data: Vec = (0..1000).collect(); + let result = (&data[..]) + .into_par_iter() + .with_pool(&mut pool) + .any(|&&x| x == 42); + assert!(result); + } + + #[test] + fn any_false() { + let mut pool = spawn(hw_threads()); + let data: Vec = (0..1000).collect(); + let result = (&data[..]) + .into_par_iter() + .with_pool(&mut pool) + .any(|&&x| x >= 2000); + assert!(!result); + } + + #[test] + fn any_empty() { + let mut pool = spawn(hw_threads()); + let data: Vec = vec![]; + let result = (&data[..]) + .into_par_iter() + .with_pool(&mut pool) + .any(|&&_x| true); + assert!(!result); + } + + #[test] + fn all_true() { + let mut pool = spawn(hw_threads()); + let data: Vec = (0..1000).collect(); + let result = (&data[..]) + .into_par_iter() + .with_pool(&mut pool) + .all(|&&x| x < 2000); + assert!(result); + } + + #[test] + fn all_false() { + let mut pool = spawn(hw_threads()); + let data: Vec = (0..1000).collect(); + let result = (&data[..]) + .into_par_iter() + .with_pool(&mut pool) + .all(|&&x| x < 500); + assert!(!result); + } + + #[test] + fn all_empty() { + let mut pool = spawn(hw_threads()); + let data: Vec = vec![]; + let result = (&data[..]) + .into_par_iter() + .with_pool(&mut pool) + .all(|&&_x| false); + assert!(result); // vacuous truth + } + + #[test] + fn try_fold_with_scratch_success() { + let mut pool = spawn(hw_threads()); + let data: Vec = (0..1000).collect(); + let mut scratch: Vec> = + (0..pool.threads()).map(|_| CacheAligned(0)).collect(); + + let result = (&data[..]) + .into_par_iter() + .with_pool(&mut pool) + .try_fold_with_scratch(scratch.as_mut_slice(), |acc, &value, _| { + acc.0 += value; + Ok::<(), &str>(()) + }); + + assert!(result.is_ok()); + let total: u64 = scratch.iter().map(|x| x.0).sum(); + assert_eq!(total, data.iter().sum()); + } + + #[test] + fn try_fold_with_scratch_early_exit() { + let mut pool = spawn(hw_threads()); + let data: Vec = (0..1000).collect(); + let mut scratch: Vec> = + (0..pool.threads()).map(|_| CacheAligned(0)).collect(); + + let result = (&data[..]) + .into_par_iter() + .with_pool(&mut pool) + .try_fold_with_scratch(scratch.as_mut_slice(), |acc, &value, _| { + if value >= 500 { + Err(value) + } else { + acc.0 += value; + Ok(()) + } + }); + + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(err >= 500 && err < 1000); + } } diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index 908d464..d01fd44 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -1,6 +1,6 @@ # Shared logic for setting target properties for tests and examples -function (set_target_properties_for_fork_union_script target_name) - target_link_libraries(${target_name} PRIVATE fork_union) +function (set_target_properties_for_forkunion_script target_name) + target_link_libraries(${target_name} PRIVATE forkunion) if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") # This warning annoys us by reminding, that before GCC 4.6 the ABI for passing objects aligned to โ‰ฅ32-bytes was @@ -12,9 +12,8 @@ function (set_target_properties_for_fork_union_script target_name) if (CMAKE_BUILD_TYPE STREQUAL "Release") message(STATUS "Enabling optimizations for ${target_name}") target_compile_options( - ${target_name} - PRIVATE $<$:-O3 -march=native -mtune=native> - $<$:/O2 /Ob3> + ${target_name} PRIVATE $<$:-O3 -march=native -mtune=native> + $<$:/O2 /Ob3> ) endif () @@ -43,7 +42,7 @@ set(TEST_SOURCES test.cpp) set(CXX_STANDARDS 17 20 23) foreach (STD IN LISTS CXX_STANDARDS) # Derive a unique target name - set(TGT fork_union_test_cpp${STD}) + set(TGT forkunion_test_cpp${STD}) # Create the executable add_executable(${TGT} ${TEST_SOURCES}) @@ -57,7 +56,7 @@ foreach (STD IN LISTS CXX_STANDARDS) # register it as a CTest test add_test(NAME ${TGT} COMMAND ${TGT}) - set_target_properties_for_fork_union_script(${TGT}) + set_target_properties_for_forkunion_script(${TGT}) # Link against `libatomic` for Linux toolchains that might need it if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang") @@ -68,39 +67,101 @@ foreach (STD IN LISTS CXX_STANDARDS) endforeach () +enable_language(C) + +# Standard C11 test +if (CMAKE_C_COMPILER_ID STREQUAL "MSVC") + message(STATUS "Skipping C11 C API tests on MSVC due to missing support") +else () + add_executable(forkunion_test_c11 test.c) + set_target_properties( + forkunion_test_c11 + PROPERTIES C_STANDARD 11 + C_STANDARD_REQUIRED ON + C_EXTENSIONS OFF + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR} + ) + target_link_libraries(forkunion_test_c11 PRIVATE forkunion_static) + add_test(NAME forkunion_test_c11 COMMAND forkunion_test_c11) + + if (CMAKE_C_COMPILER_ID STREQUAL "GNU" OR CMAKE_C_COMPILER_ID STREQUAL "Clang") + if (UNIX AND NOT APPLE) + target_link_libraries(forkunion_test_c11 PRIVATE -latomic) + endif () + endif () +endif () + +# GCC nested functions extension test +if (CMAKE_C_COMPILER_ID STREQUAL "GNU") + add_executable(forkunion_test_c_gcc_nested test.c) + set_target_properties( + forkunion_test_c_gcc_nested + PROPERTIES C_STANDARD 11 + C_STANDARD_REQUIRED ON + C_EXTENSIONS ON + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR} + ) + target_link_libraries(forkunion_test_c_gcc_nested PRIVATE forkunion_static) + add_test(NAME forkunion_test_c_gcc_nested COMMAND forkunion_test_c_gcc_nested) + + if (UNIX AND NOT APPLE) + target_link_libraries(forkunion_test_c_gcc_nested PRIVATE -latomic) + endif () +endif () + +# Clang blocks extension test +if (CMAKE_C_COMPILER_ID STREQUAL "Clang" OR CMAKE_C_COMPILER_ID STREQUAL "AppleClang") + add_executable(forkunion_test_c_clang_blocks test.c) + set_target_properties( + forkunion_test_c_clang_blocks + PROPERTIES C_STANDARD 11 + C_STANDARD_REQUIRED ON + C_EXTENSIONS ON + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR} + ) + target_compile_options(forkunion_test_c_clang_blocks PRIVATE -fblocks) + target_link_libraries(forkunion_test_c_clang_blocks PRIVATE forkunion_static) + add_test(NAME forkunion_test_c_clang_blocks COMMAND forkunion_test_c_clang_blocks) + + if (APPLE) + # Apple's `libSystem` already ships the Blocks runtime, so no extra linkage is required + elseif (UNIX) + target_link_libraries(forkunion_test_c_clang_blocks PRIVATE -latomic -lBlocksRuntime) + endif () +endif () + # Include the N-body benchmark -add_executable(fork_union_nbody nbody.cpp) -target_link_libraries(fork_union_nbody PRIVATE fork_union) +add_executable(forkunion_nbody nbody.cpp) +target_link_libraries(forkunion_nbody PRIVATE forkunion) set_target_properties( - fork_union_nbody + forkunion_nbody PROPERTIES CXX_STANDARD 20 CXX_STANDARD_REQUIRED ON CXX_EXTENSIONS OFF ) if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - message(STATUS "Enabling OpenMP for fork_union_nbody on GCC") - target_compile_options(fork_union_nbody PRIVATE -fopenmp -flto -ffast-math) - target_link_options(fork_union_nbody PRIVATE -fopenmp) + message(STATUS "Enabling OpenMP for forkunion_nbody on GCC") + target_compile_options(forkunion_nbody PRIVATE -fopenmp -flto -ffast-math) + target_link_options(forkunion_nbody PRIVATE -fopenmp) elseif (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang") find_program(BREW_EXECUTABLE brew) if (BREW_EXECUTABLE) execute_process( COMMAND ${BREW_EXECUTABLE} --prefix libomp OUTPUT_VARIABLE LIBOMP_PREFIX - OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_QUIET + OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET ) if (LIBOMP_PREFIX AND EXISTS "${LIBOMP_PREFIX}/include/omp.h") - message(STATUS "Enabling OpenMP for fork_union_nbody on Clang with Homebrew libomp") - target_include_directories(fork_union_nbody PRIVATE ${LIBOMP_PREFIX}/include) - target_link_directories(fork_union_nbody PRIVATE ${LIBOMP_PREFIX}/lib) - target_compile_options(fork_union_nbody PRIVATE -Xclang -fopenmp) - target_link_libraries(fork_union_nbody PRIVATE omp) + message(STATUS "Enabling OpenMP for forkunion_nbody on Clang with Homebrew libomp") + target_include_directories(forkunion_nbody PRIVATE ${LIBOMP_PREFIX}/include) + target_link_directories(forkunion_nbody PRIVATE ${LIBOMP_PREFIX}/lib) + target_compile_options(forkunion_nbody PRIVATE -Xclang -fopenmp) + target_link_libraries(forkunion_nbody PRIVATE omp) else () - message(STATUS "Homebrew libomp not found - OpenMP disabled for fork_union_nbody") + message(STATUS "Homebrew libomp not found - OpenMP disabled for forkunion_nbody") endif () else () - message(STATUS "Homebrew not found - OpenMP disabled for fork_union_nbody") + message(STATUS "Homebrew not found - OpenMP disabled for forkunion_nbody") endif () endif () -set_target_properties_for_fork_union_script(fork_union_nbody) +set_target_properties_for_forkunion_script(forkunion_nbody) diff --git a/scripts/build.zig b/scripts/build.zig new file mode 100644 index 0000000..37b0831 --- /dev/null +++ b/scripts/build.zig @@ -0,0 +1,56 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + const enable_numa = b.option(bool, "numa", "Enable NUMA support (Linux only)") orelse + (target.result.os.tag == .linux); + + // Get the forkunion module and artifact from parent + const forkunion_dep = b.dependency("forkunion", .{ + .target = target, + .optimize = optimize, + .numa = enable_numa, + }); + const forkunion_module = forkunion_dep.module("forkunion"); + const forkunion_artifact = forkunion_dep.artifact("forkunion"); + + // N-body benchmark executable + const nbody = b.addExecutable(.{ + .name = "nbody", + .root_module = b.createModule(.{ + .root_source_file = b.path("nbody.zig"), + .target = target, + .optimize = optimize, + }), + }); + + nbody.linkLibC(); + nbody.linkLibCpp(); + nbody.linkLibrary(forkunion_artifact); + if (target.result.os.tag == .linux) { + nbody.root_module.linkSystemLibrary("pthread", .{}); + if (enable_numa) { + nbody.root_module.linkSystemLibrary("numa", .{}); + } + } + nbody.root_module.addImport("forkunion", forkunion_module); + + // Add benchmark dependencies + if (b.lazyDependency("libxev", .{ + .target = target, + .optimize = optimize, + })) |libxev_dep| { + nbody.root_module.addImport("xev", libxev_dep.module("xev")); + } + + b.installArtifact(nbody); + + const run_nbody = b.addRunArtifact(nbody); + if (b.args) |args| { + run_nbody.addArgs(args); + } + + const run_step = b.step("run", "Run N-body benchmark"); + run_step.dependOn(&run_nbody.step); +} diff --git a/scripts/build.zig.zon b/scripts/build.zig.zon new file mode 100644 index 0000000..54286ab --- /dev/null +++ b/scripts/build.zig.zon @@ -0,0 +1,23 @@ +.{ + .name = .nbody, + .version = "0.1.0", + .fingerprint = 0x1accea9f530b9d5f, + .minimum_zig_version = "0.15.0", + + .dependencies = .{ + .forkunion = .{ + .path = "..", + }, + .libxev = .{ + .url = "git+https://github.com/mitchellh/libxev#96a08cd8ae3bbf291f07b7cc3de8b401cb60df5e", + .hash = "12209dc002b8a58b69cd8e55034a00b4124b5d9abc8b1c1eaecbd31a39282caae267", + .lazy = true, + }, + }, + + .paths = .{ + "build.zig", + "build.zig.zon", + "nbody.zig", + }, +} diff --git a/scripts/nbody.cpp b/scripts/nbody.cpp index b2236d1..c5fcd36 100644 --- a/scripts/nbody.cpp +++ b/scripts/nbody.cpp @@ -1,5 +1,5 @@ /** - * @brief Demo app: N-Body simulation with Fork Union and OpenMP. + * @brief Demo app: N-Body simulation with ForkUnion and OpenMP. * @author Ash Vardanian * @file nbody.cpp * @@ -7,29 +7,29 @@ * * - `NBODY_COUNT` - number of bodies in the simulation (default: number of threads). * - `NBODY_ITERATIONS` - number of iterations to run the simulation (default: 1000). - * - `NBODY_BACKEND` - backend to use for the simulation (default: `fork_union_static`). + * - `NBODY_BACKEND` - backend to use for the simulation (default: `forkunion_static`). * - `NBODY_THREADS` - number of threads to use for the simulation (default: number of hardware threads). * - * The backends include: `fork_union_static`, `fork_union_dynamic`, `openmp_static`, and `openmp_dynamic`. + * The backends include: `forkunion_static`, `forkunion_dynamic`, `openmp_static`, and `openmp_dynamic`. * To compile and run on all cores in Linux: * * @code{.sh} * cmake -B build_release -D CMAKE_BUILD_TYPE=Release * cmake --build build_release --config Release - * NBODY_COUNT=128 NBODY_THREADS=$(nproc) build_release/fork_union_nbody + * NBODY_COUNT=128 NBODY_THREADS=$(nproc) build_release/forkunion_nbody * @endcode * * The default profiling scheme is to 1M iterations for 128 particles on each backend: * * @code{.sh} * time NBODY_COUNT=128 NBODY_THREADS=$(nproc) NBODY_ITERATIONS=1000000 \ - * NBODY_BACKEND=openmp_static build_release/fork_union_nbody + * NBODY_BACKEND=openmp_static build_release/forkunion_nbody * time NBODY_COUNT=128 NBODY_THREADS=$(nproc) NBODY_ITERATIONS=1000000 \ - * NBODY_BACKEND=openmp_dynamic build_release/fork_union_nbody + * NBODY_BACKEND=openmp_dynamic build_release/forkunion_nbody * time NBODY_COUNT=128 NBODY_THREADS=$(nproc) NBODY_ITERATIONS=1000000 \ - * NBODY_BACKEND=fork_union_static build_release/fork_union_nbody + * NBODY_BACKEND=forkunion_static build_release/forkunion_nbody * time NBODY_COUNT=128 NBODY_THREADS=$(nproc) NBODY_ITERATIONS=1000000 \ - * NBODY_BACKEND=fork_union_dynamic build_release/fork_union_nbody + * NBODY_BACKEND=forkunion_dynamic build_release/forkunion_nbody * @endcode * * On macOS, you may need to install OpenMP support via Homebrew: @@ -43,7 +43,7 @@ * -D CMAKE_EXE_LINKER_FLAGS="-L$(brew --prefix libomp)/lib" * cmake --build build_release --config Release * NBODY_COUNT=128 NBODY_THREADS=$(sysctl -n hw.logicalcpu) NBODY_ITERATIONS=1000000 \ - * NBODY_BACKEND=fork_union_static build_release/fork_union_nbody + * NBODY_BACKEND=forkunion_static build_release/forkunion_nbody * @endcode */ #include // `std::vector` @@ -63,9 +63,9 @@ #endif #endif -#include +#include -namespace fu = ashvardanian::fork_union; +namespace fu = ashvardanian::forkunion; #if defined(__GNUC__) || defined(__clang__) #define _FU_RESTRICT __restrict__ @@ -160,8 +160,8 @@ void iteration_openmp_dynamic(FU_MAYBE_UNUSED_ body_t *_FU_RESTRICT bodies, using pool_t = fu::basic_pool, fu::standard_yield_t>; -void iteration_fork_union_static(pool_t &pool, body_t *_FU_RESTRICT bodies, vector3_t *_FU_RESTRICT forces, - std::size_t n) noexcept { +void iteration_forkunion_static(pool_t &pool, body_t *_FU_RESTRICT bodies, vector3_t *_FU_RESTRICT forces, + std::size_t n) noexcept { pool.for_n(n, [=](std::size_t i) noexcept { vector3_t f {0.0, 0.0, 0.0}; for (std::size_t j = 0; j < n; ++j) f += gravitational_force(bodies[i], bodies[j]); @@ -170,8 +170,8 @@ void iteration_fork_union_static(pool_t &pool, body_t *_FU_RESTRICT bodies, vect pool.for_n(n, [=](std::size_t i) noexcept { apply_force(bodies[i], forces[i]); }); } -void iteration_fork_union_dynamic(pool_t &pool, body_t *_FU_RESTRICT bodies, vector3_t *_FU_RESTRICT forces, - std::size_t n) noexcept { +void iteration_forkunion_dynamic(pool_t &pool, body_t *_FU_RESTRICT bodies, vector3_t *_FU_RESTRICT forces, + std::size_t n) noexcept { pool.for_n_dynamic(n, [=](std::size_t i) noexcept { vector3_t f {0.0, 0.0, 0.0}; for (std::size_t j = 0; j < n; ++j) f += gravitational_force(bodies[i], bodies[j]); @@ -185,8 +185,8 @@ using linux_numa_bodies_allocator_t = fu::linux_numa_allocator; using linux_numa_bodies_t = std::vector; using linux_distributed_pool_t = fu::linux_distributed_pool; -std::vector make_buffers_for_fork_union_numa(linux_distributed_pool_t &pool, - std::size_t n) noexcept { +std::vector make_buffers_for_forkunion_numa(linux_distributed_pool_t &pool, + std::size_t n) noexcept { fu::numa_topology_t const &topology = pool.topology(); std::size_t const numa_nodes_count = topology.nodes_count(); @@ -201,9 +201,9 @@ std::vector make_buffers_for_fork_union_numa(linux_distribu return result; } -void iteration_fork_union_numa_static(linux_distributed_pool_t &pool, body_t *_FU_RESTRICT bodies, - vector3_t *_FU_RESTRICT forces, std::size_t n, - body_t **_FU_RESTRICT bodies_numa_copies) noexcept { +void iteration_forkunion_numa_static(linux_distributed_pool_t &pool, body_t *_FU_RESTRICT bodies, + vector3_t *_FU_RESTRICT forces, std::size_t n, + body_t **_FU_RESTRICT bodies_numa_copies) noexcept { using colocated_prong_t = typename linux_distributed_pool_t::prong_t; @@ -236,13 +236,13 @@ void iteration_fork_union_numa_static(linux_distributed_pool_t &pool, body_t *_F pool.for_n(n, [=](std::size_t i) noexcept { apply_force(bodies[i], forces[i]); }); } -void iteration_fork_union_numa_dynamic(linux_distributed_pool_t &pool, body_t *_FU_RESTRICT bodies, - vector3_t *_FU_RESTRICT forces, std::size_t n, - body_t **_FU_RESTRICT bodies_numa_copies) noexcept { +void iteration_forkunion_numa_dynamic(linux_distributed_pool_t &pool, body_t *_FU_RESTRICT bodies, + vector3_t *_FU_RESTRICT forces, std::size_t n, + body_t **_FU_RESTRICT bodies_numa_copies) noexcept { using colocated_prong_t = typename linux_distributed_pool_t::prong_t; - // This expressions is same as in `iteration_fork_union_numa_static` static version: + // This expressions is same as in `iteration_forkunion_numa_static` static version: pool.for_threads([&](auto thread_index) noexcept { std::size_t const numa_node_index = pool.thread_colocation(thread_index); std::size_t const threads_next_to_numa_node = pool.threads_count(numa_node_index); @@ -274,7 +274,7 @@ void iteration_fork_union_numa_dynamic(linux_distributed_pool_t &pool, body_t *_ #pragma endregion - Backends int main(void) { - std::printf("Welcome to the Fork Union N-Body simulation!\n"); + std::printf("Welcome to the ForkUnion N-Body simulation!\n"); // Helper function to safely get environment variables auto safe_getenv = [](char const *name) -> char const * { @@ -303,7 +303,7 @@ int main(void) { std::size_t n = (n_ull > size_max) ? size_max : static_cast(n_ull); std::size_t const iterations = (iterations_ull > size_max) ? size_max : static_cast(iterations_ull); std::size_t threads = (threads_ull > size_max) ? size_max : static_cast(threads_ull); - std::string_view const backend = backend_str ? backend_str : "fork_union_static"; + std::string_view const backend = backend_str ? backend_str : "forkunion_static"; if (threads == 0) threads = std::thread::hardware_concurrency(); if (n == 0) n = threads; @@ -338,24 +338,23 @@ int main(void) { } #endif - // Every other configuration uses Fork Union + // Every other configuration uses ForkUnion pool_t pool; - if (backend == "fork_union_static") { + if (backend == "forkunion_static") { if (!pool.try_spawn(threads)) { std::fprintf(stderr, "Failed to spawn thread pool\n"); return EXIT_FAILURE; } for (std::size_t i = 0; i < iterations; ++i) // - iteration_fork_union_static(pool, bodies.data(), forces.data(), n); + iteration_forkunion_static(pool, bodies.data(), forces.data(), n); return EXIT_SUCCESS; } - if (backend == "fork_union_dynamic") { + if (backend == "forkunion_dynamic") { if (!pool.try_spawn(threads)) { std::fprintf(stderr, "Failed to spawn thread pool\n"); return EXIT_FAILURE; } - for (std::size_t i = 0; i < iterations; ++i) - iteration_fork_union_dynamic(pool, bodies.data(), forces.data(), n); + for (std::size_t i = 0; i < iterations; ++i) iteration_forkunion_dynamic(pool, bodies.data(), forces.data(), n); return EXIT_SUCCESS; } @@ -367,26 +366,26 @@ int main(void) { } linux_distributed_pool_t numa_pool(std::move(topology)); - std::vector bodies_numa_arrays = make_buffers_for_fork_union_numa(numa_pool, n); + std::vector bodies_numa_arrays = make_buffers_for_forkunion_numa(numa_pool, n); std::vector bodies_numa_buffers(bodies_numa_arrays.size()); for (std::size_t i = 0; i < bodies_numa_arrays.size(); ++i) bodies_numa_buffers[i] = bodies_numa_arrays[i].data(); - if (backend == "fork_union_numa_static") { + if (backend == "forkunion_numa_static") { if (!numa_pool.try_spawn(threads)) { std::fprintf(stderr, "Failed to spawn NUMA thread pools\n"); return EXIT_FAILURE; } for (std::size_t i = 0; i < iterations; ++i) - iteration_fork_union_numa_static(numa_pool, bodies.data(), forces.data(), n, bodies_numa_buffers.data()); + iteration_forkunion_numa_static(numa_pool, bodies.data(), forces.data(), n, bodies_numa_buffers.data()); return EXIT_SUCCESS; } - if (backend == "fork_union_numa_dynamic") { + if (backend == "forkunion_numa_dynamic") { if (!numa_pool.try_spawn(threads)) { std::fprintf(stderr, "Failed to spawn NUMA thread pools\n"); return EXIT_FAILURE; } for (std::size_t i = 0; i < iterations; ++i) - iteration_fork_union_numa_dynamic(numa_pool, bodies.data(), forces.data(), n, bodies_numa_buffers.data()); + iteration_forkunion_numa_dynamic(numa_pool, bodies.data(), forces.data(), n, bodies_numa_buffers.data()); return EXIT_SUCCESS; } #endif // FU_ENABLE_NUMA diff --git a/scripts/nbody.rs b/scripts/nbody.rs index 5138390..f27db04 100644 --- a/scripts/nbody.rs +++ b/scripts/nbody.rs @@ -1,14 +1,14 @@ -//! Demo app: N-Body simulation with Fork Union and Rayon. +//! Demo app: N-Body simulation with ForkUnion and Rayon. //! //! To control the script, several environment variables are used: //! //! - `NBODY_COUNT` - number of bodies in the simulation (default: number of threads). //! - `NBODY_ITERATIONS` - number of iterations to run the simulation (default: 1000). -//! - `NBODY_BACKEND` - backend to use for the simulation (default: `fork_union_static`). +//! - `NBODY_BACKEND` - backend to use for the simulation (default: `forkunion_static`). //! - `NBODY_THREADS` - number of threads to use for the simulation (default: number of hardware threads). //! -//! The backends include: `fork_union_static`, `fork_union_dynamic`, `fork_union_iter_static`, -//! `fork_union_iter_dynamic`, `rayon_static`, `rayon_dynamic`, and `tokio`. To compile and run: +//! The backends include: `forkunion_static`, `forkunion_dynamic`, `forkunion_iter_static`, +//! `forkunion_iter_dynamic`, `rayon_static`, `rayon_dynamic`, and `tokio`. To compile and run: //! //! ```sh //! cargo run --example nbody --release @@ -27,25 +27,25 @@ //! time NBODY_COUNT=128 NBODY_THREADS=$(nproc) NBODY_ITERATIONS=1000000 \ //! NBODY_BACKEND=rayon_dynamic target/release/examples/nbody //! time NBODY_COUNT=128 NBODY_THREADS=$(nproc) NBODY_ITERATIONS=1000000 \ -//! NBODY_BACKEND=fork_union_static target/release/examples/nbody +//! NBODY_BACKEND=forkunion_static target/release/examples/nbody //! time NBODY_COUNT=128 NBODY_THREADS=$(nproc) NBODY_ITERATIONS=1000000 \ -//! NBODY_BACKEND=fork_union_dynamic target/release/examples/nbody +//! NBODY_BACKEND=forkunion_dynamic target/release/examples/nbody //! time NBODY_COUNT=128 NBODY_THREADS=$(nproc) NBODY_ITERATIONS=1000000 \ -//! NBODY_BACKEND=fork_union_iter_static target/release/examples/nbody +//! NBODY_BACKEND=forkunion_iter_static target/release/examples/nbody //! time NBODY_COUNT=128 NBODY_THREADS=$(nproc) NBODY_ITERATIONS=1000000 \ -//! NBODY_BACKEND=fork_union_iter_dynamic target/release/examples/nbody +//! NBODY_BACKEND=forkunion_iter_dynamic target/release/examples/nbody //! time NBODY_COUNT=128 NBODY_THREADS=$(nproc) NBODY_ITERATIONS=1000000 \ //! NBODY_BACKEND=tokio target/release/examples/nbody //! //! # macOS benchmarks (use sysctl -n hw.logicalcpu for CPU count) //! time NBODY_COUNT=128 NBODY_THREADS=$(sysctl -n hw.logicalcpu) NBODY_ITERATIONS=1000000 \ -//! NBODY_BACKEND=fork_union_iter_static target/release/examples/nbody +//! NBODY_BACKEND=forkunion_iter_static target/release/examples/nbody //! ``` use rand::{rng, Rng}; use std::env; use std::error::Error; -use fork_union as fu; +use forkunion as fu; use rayon::{prelude::*, ThreadPool, ThreadPoolBuilder}; use tokio::task::JoinSet; @@ -357,7 +357,7 @@ fn main() -> Result<(), Box> { .ok() .and_then(|v| v.parse().ok()) .unwrap_or(1_000); - let backend = env::var("NBODY_BACKEND").unwrap_or_else(|_| "fork_union_static".into()); + let backend = env::var("NBODY_BACKEND").unwrap_or_else(|_| "forkunion_static".into()); let threads = env::var("NBODY_THREADS") .ok() .and_then(|v| v.parse().ok()) @@ -396,28 +396,28 @@ fn main() -> Result<(), Box> { // Run the chosen backend match backend.as_str() { - "fork_union_static" => { + "forkunion_static" => { let mut pool = fu::ThreadPool::try_spawn(threads) .unwrap_or_else(|e| panic!("Failed to start Fork-Union pool: {e}")); for _ in 0..iters { iteration_fu_static(&mut pool, &mut bodies, &mut forces); } } - "fork_union_dynamic" => { + "forkunion_dynamic" => { let mut pool = fu::ThreadPool::try_spawn(threads) .unwrap_or_else(|e| panic!("Failed to start Fork-Union pool: {e}")); for _ in 0..iters { iteration_fu_dynamic(&mut pool, &mut bodies, &mut forces); } } - "fork_union_iter_static" => { + "forkunion_iter_static" => { let mut pool = fu::ThreadPool::try_spawn(threads) .unwrap_or_else(|e| panic!("Failed to start Fork-Union pool: {e}")); for _ in 0..iters { iteration_fu_iter_static(&mut pool, &mut bodies, &mut forces); } } - "fork_union_iter_dynamic" => { + "forkunion_iter_dynamic" => { let mut pool = fu::ThreadPool::try_spawn(threads) .unwrap_or_else(|e| panic!("Failed to start Fork-Union pool: {e}")); for _ in 0..iters { diff --git a/scripts/nbody.zig b/scripts/nbody.zig new file mode 100644 index 0000000..7727ca6 --- /dev/null +++ b/scripts/nbody.zig @@ -0,0 +1,422 @@ +//! N-Body simulation benchmark comparing different parallelism libraries +//! +//! Compares synchronization overhead of different thread pool implementations: +//! - forkunion_static: Static work division (N tasks pre-divided into thread slices) +//! - forkunion_dynamic: Dynamic work-stealing (ForkUnion's work-stealing scheduler) +//! - std: Static work division (std.Thread.Pool with manual slicing) +//! - libxev: Dynamic lock-free queue (Mitchell Hashimoto's lock-free thread pool) +//! +//! Environment variables: +//! - NBODY_COUNT: number of bodies (default: number of threads) +//! - NBODY_ITERATIONS: number of iterations (default: 1000) +//! - NBODY_BACKEND: forkunion_static, forkunion_dynamic, std, libxev +//! - NBODY_THREADS: number of threads (default: CPU count) +//! +//! Build and run from scripts/ directory: +//! ```sh +//! cd scripts +//! zig build -Doptimize=ReleaseFast +//! time NBODY_COUNT=128 NBODY_ITERATIONS=1000000 NBODY_BACKEND=forkunion_static ./zig-out/bin/nbody +//! time NBODY_COUNT=128 NBODY_ITERATIONS=1000000 NBODY_BACKEND=forkunion_dynamic ./zig-out/bin/nbody +//! time NBODY_COUNT=128 NBODY_ITERATIONS=1000000 NBODY_BACKEND=libxev ./zig-out/bin/nbody +//! time NBODY_COUNT=128 NBODY_ITERATIONS=1000000 NBODY_BACKEND=std ./zig-out/bin/nbody +//! ``` + +const std = @import("std"); +const fu = @import("forkunion"); +const xev = @import("xev"); + +// Physical constants +const G: f32 = 6.674e-11; +const DT: f32 = 0.01; +const SOFTEN: f32 = 1.0e-9; + +const Vector3 = struct { + x: f32 = 0, + y: f32 = 0, + z: f32 = 0, + + fn addAssign(self: *Vector3, other: Vector3) void { + self.x += other.x; + self.y += other.y; + self.z += other.z; + } +}; + +const Body = struct { + position: Vector3 = .{}, + velocity: Vector3 = .{}, + mass: f32 = 0, +}; + +/// Fast reciprocal square root (Quake-style with one Newton iteration) +inline fn fastRsqrt(x: f32) f32 { + const i = 0x5f3759df - (@as(u32, @bitCast(x)) >> 1); + var y = @as(f32, @bitCast(i)); + const x2 = 0.5 * x; + y *= 1.5 - x2 * y * y; + return y; +} + +inline fn gravitationalForce(bi: *const Body, bj: *const Body) Vector3 { + const dx = bj.position.x - bi.position.x; + const dy = bj.position.y - bi.position.y; + const dz = bj.position.z - bi.position.z; + const l2 = dx * dx + dy * dy + dz * dz + SOFTEN; + const inv = fastRsqrt(l2); + const inv3 = inv * inv * inv; + const mag = G * bi.mass * bj.mass * inv3; + return .{ + .x = mag * dx, + .y = mag * dy, + .z = mag * dz, + }; +} + +inline fn applyForce(b: *Body, f: *const Vector3) void { + b.velocity.x += f.x / b.mass * DT; + b.velocity.y += f.y / b.mass * DT; + b.velocity.z += f.z / b.mass * DT; + + b.position.x += b.velocity.x * DT; + b.position.y += b.velocity.y * DT; + b.position.z += b.velocity.z * DT; +} + +// ============================================================================ +// ForkUnion Kernels +// ============================================================================ + +fn iterationForkUnionStatic(pool: *fu.Pool, bodies: []Body, forces: []Vector3) void { + const n = bodies.len; + + // First pass: calculate forces + const CalcContext = struct { + bodies_ptr: [*]const Body, + forces_ptr: [*]Vector3, + n: usize, + }; + + pool.forN(n, struct { + fn calc(prong: fu.Prong, ctx: CalcContext) void { + const bi = &ctx.bodies_ptr[prong.task_index]; + var acc = Vector3{}; + + for (0..ctx.n) |j| { + acc.addAssign(gravitationalForce(bi, &ctx.bodies_ptr[j])); + } + ctx.forces_ptr[prong.task_index] = acc; + } + }.calc, CalcContext{ + .bodies_ptr = bodies.ptr, + .forces_ptr = forces.ptr, + .n = n, + }); + + // Second pass: apply forces + const ApplyContext = struct { + bodies_ptr: [*]Body, + forces_ptr: [*]const Vector3, + }; + + pool.forN(n, struct { + fn apply(prong: fu.Prong, ctx: ApplyContext) void { + applyForce(&ctx.bodies_ptr[prong.task_index], &ctx.forces_ptr[prong.task_index]); + } + }.apply, ApplyContext{ + .bodies_ptr = bodies.ptr, + .forces_ptr = forces.ptr, + }); +} + +fn iterationForkUnionDynamic(pool: *fu.Pool, bodies: []Body, forces: []Vector3) void { + const n = bodies.len; + + // First pass: calculate forces + const CalcContext = struct { + bodies_ptr: [*]const Body, + forces_ptr: [*]Vector3, + n: usize, + }; + + pool.forNDynamic(n, struct { + fn calc(prong: fu.Prong, ctx: CalcContext) void { + const bi = &ctx.bodies_ptr[prong.task_index]; + var acc = Vector3{}; + + for (0..ctx.n) |j| { + acc.addAssign(gravitationalForce(bi, &ctx.bodies_ptr[j])); + } + ctx.forces_ptr[prong.task_index] = acc; + } + }.calc, CalcContext{ + .bodies_ptr = bodies.ptr, + .forces_ptr = forces.ptr, + .n = n, + }); + + // Second pass: apply forces + const ApplyContext = struct { + bodies_ptr: [*]Body, + forces_ptr: [*]const Vector3, + }; + + pool.forNDynamic(n, struct { + fn apply(prong: fu.Prong, ctx: ApplyContext) void { + applyForce(&ctx.bodies_ptr[prong.task_index], &ctx.forces_ptr[prong.task_index]); + } + }.apply, ApplyContext{ + .bodies_ptr = bodies.ptr, + .forces_ptr = forces.ptr, + }); +} + +// ============================================================================ +// std.Thread.Pool Backend (Static Work Division) +// Divides N tasks into equal slices per thread for static work distribution. +// ============================================================================ + +fn iterationStdPool(pool: *std.Thread.Pool, bodies: []Body, forces: []Vector3, n_threads: usize) !void { + const n = bodies.len; + + // First pass: calculate forces + { + var wg: std.Thread.WaitGroup = .{}; + const chunk_size = (n + n_threads - 1) / n_threads; + + for (0..n_threads) |thread_id| { + const start = thread_id * chunk_size; + if (start >= n) break; + const end = @min(start + chunk_size, n); + + pool.spawnWg(&wg, struct { + fn calc(bodies_slice: []const Body, forces_slice: []Vector3, range_start: usize, range_end: usize) void { + for (range_start..range_end) |i| { + const bi = &bodies_slice[i]; + var acc = Vector3{}; + for (bodies_slice) |*bj| { + acc.addAssign(gravitationalForce(bi, bj)); + } + forces_slice[i] = acc; + } + } + }.calc, .{ bodies, forces, start, end }); + } + pool.waitAndWork(&wg); + } + + // Second pass: apply forces + { + var wg: std.Thread.WaitGroup = .{}; + const chunk_size = (n + n_threads - 1) / n_threads; + + for (0..n_threads) |thread_id| { + const start = thread_id * chunk_size; + if (start >= n) break; + const end = @min(start + chunk_size, n); + + pool.spawnWg(&wg, struct { + fn apply(bodies_slice: []Body, forces_slice: []const Vector3, range_start: usize, range_end: usize) void { + for (range_start..range_end) |i| { + applyForce(&bodies_slice[i], &forces_slice[i]); + } + } + }.apply, .{ bodies, forces, start, end }); + } + pool.waitAndWork(&wg); + } +} + +// ============================================================================ +// libxev ThreadPool Backend (Lock-Free Queue - Dynamic) +// Uses libxev's lock-free thread pool with batch task scheduling. Creates N +// tasks, batches them, and relies on the framework's lock-free queue for +// dynamic work distribution across workers. +// ============================================================================ + +fn iterationLibxev(pool: *xev.ThreadPool, bodies: []Body, forces: []Vector3, allocator: std.mem.Allocator) !void { + const n = bodies.len; + + // Task context for force calculation + const CalcContext = struct { + task: xev.ThreadPool.Task, + bodies: []const Body, + forces: []Vector3, + idx: usize, + done: *std.atomic.Value(usize), + + fn run(task_ptr: *xev.ThreadPool.Task) void { + const ctx: *@This() = @fieldParentPtr("task", task_ptr); + const bi = &ctx.bodies[ctx.idx]; + var acc = Vector3{}; + for (ctx.bodies) |*bj| { + acc.addAssign(gravitationalForce(bi, bj)); + } + ctx.forces[ctx.idx] = acc; + _ = ctx.done.fetchAdd(1, .monotonic); + } + }; + + // Allocate contexts for force calculation + var calc_contexts = try allocator.alloc(CalcContext, n); + defer allocator.free(calc_contexts); + var calc_done = std.atomic.Value(usize).init(0); + + for (0..n) |i| { + calc_contexts[i] = .{ + .task = .{ .callback = CalcContext.run }, + .bodies = bodies, + .forces = forces, + .idx = i, + .done = &calc_done, + }; + } + + // Schedule all force calculation tasks + var calc_batch = xev.ThreadPool.Batch{}; + for (calc_contexts) |*ctx| { + calc_batch.push(xev.ThreadPool.Batch.from(&ctx.task)); + } + pool.schedule(calc_batch); + + // Wait for completion + while (calc_done.load(.acquire) < n) { + std.atomic.spinLoopHint(); + } + + // Task context for applying forces + const ApplyContext = struct { + task: xev.ThreadPool.Task, + bodies: []Body, + forces: []const Vector3, + idx: usize, + done: *std.atomic.Value(usize), + + fn run(task_ptr: *xev.ThreadPool.Task) void { + const ctx: *@This() = @fieldParentPtr("task", task_ptr); + applyForce(&ctx.bodies[ctx.idx], &ctx.forces[ctx.idx]); + _ = ctx.done.fetchAdd(1, .monotonic); + } + }; + + // Allocate contexts for applying forces + var apply_contexts = try allocator.alloc(ApplyContext, n); + defer allocator.free(apply_contexts); + var apply_done = std.atomic.Value(usize).init(0); + + for (0..n) |i| { + apply_contexts[i] = .{ + .task = .{ .callback = ApplyContext.run }, + .bodies = bodies, + .forces = forces, + .idx = i, + .done = &apply_done, + }; + } + + // Schedule all apply force tasks + var apply_batch = xev.ThreadPool.Batch{}; + for (apply_contexts) |*ctx| { + apply_batch.push(xev.ThreadPool.Batch.from(&ctx.task)); + } + pool.schedule(apply_batch); + + // Wait for completion + while (apply_done.load(.acquire) < n) { + std.atomic.spinLoopHint(); + } +} + +// ============================================================================ +// Main +// ============================================================================ + +pub fn main() !void { + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + // Parse environment variables + const n_threads = if (std.process.getEnvVarOwned(allocator, "NBODY_THREADS")) |str| blk: { + defer allocator.free(str); + break :blk try std.fmt.parseInt(usize, str, 10); + } else |_| fu.countLogicalCores(); + + const n_iters = if (std.process.getEnvVarOwned(allocator, "NBODY_ITERATIONS")) |str| blk: { + defer allocator.free(str); + break :blk try std.fmt.parseInt(usize, str, 10); + } else |_| 1000; + + const n_bodies = if (std.process.getEnvVarOwned(allocator, "NBODY_COUNT")) |str| blk: { + defer allocator.free(str); + break :blk try std.fmt.parseInt(usize, str, 10); + } else |_| n_threads; + + const backend = if (std.process.getEnvVarOwned(allocator, "NBODY_BACKEND")) |str| + str + else |_| + "forkunion_static"; + defer if (!std.mem.eql(u8, backend, "forkunion_static")) allocator.free(backend); + + // Allocate bodies and forces + const bodies = try allocator.alloc(Body, n_bodies); + defer allocator.free(bodies); + const forces = try allocator.alloc(Vector3, n_bodies); + defer allocator.free(forces); + + // Initialize bodies + var prng = std.Random.DefaultPrng.init(@intCast(std.time.timestamp())); + const random = prng.random(); + for (bodies) |*body| { + body.position = .{ + .x = random.float(f32), + .y = random.float(f32), + .z = random.float(f32), + }; + body.velocity = .{ + .x = random.float(f32), + .y = random.float(f32), + .z = random.float(f32), + }; + body.mass = random.float(f32) * 9.0e24 + 1.0e20; // [1e20, 1e25) + } + + // Run the chosen backend + if (std.mem.eql(u8, backend, "forkunion_static")) { + var pool = try fu.Pool.init(n_threads, .inclusive); + defer pool.deinit(); + + for (0..n_iters) |_| { + iterationForkUnionStatic(&pool, bodies, forces); + } + } else if (std.mem.eql(u8, backend, "forkunion_dynamic")) { + var pool = try fu.Pool.init(n_threads, .inclusive); + defer pool.deinit(); + + for (0..n_iters) |_| { + iterationForkUnionDynamic(&pool, bodies, forces); + } + } else if (std.mem.eql(u8, backend, "std")) { + var pool: std.Thread.Pool = undefined; + try pool.init(.{ .allocator = allocator, .n_jobs = @intCast(n_threads) }); + defer pool.deinit(); + + for (0..n_iters) |_| { + try iterationStdPool(&pool, bodies, forces, n_threads); + } + } else if (std.mem.eql(u8, backend, "libxev")) { + var pool = xev.ThreadPool.init(.{ .max_threads = @intCast(n_threads) }); + defer { + pool.shutdown(); + pool.deinit(); + } + + for (0..n_iters) |_| { + try iterationLibxev(&pool, bodies, forces, allocator); + } + } else { + std.debug.print("Unknown backend: {s}\n", .{backend}); + std.debug.print("Available backends: forkunion_static, forkunion_dynamic, std, libxev\n", .{}); + return error.UnknownBackend; + } +} diff --git a/scripts/search.rs b/scripts/search.rs index 7d5d0b7..6a7190c 100644 --- a/scripts/search.rs +++ b/scripts/search.rs @@ -1,7 +1,7 @@ -//! NUMA-aware vector search implementation using Fork Union and PinnedVec with SimSIMD. +//! NUMA-aware vector search implementation using ForkUnion and PinnedVec with SimSIMD. //! //! This example demonstrates how to perform efficient similarity search across -//! multiple NUMA nodes using the PinnedVec container, Fork Union's distributed +//! multiple NUMA nodes using the PinnedVec container, ForkUnion's distributed //! thread pool capabilities, and SimSIMD for optimized distance calculations. //! //! To run this example: @@ -18,7 +18,7 @@ use rand::{rng, Rng}; use std::env; use std::time::Instant; -use fork_union as fu; +use forkunion as fu; use simsimd::{bf16, Distance, SpatialSimilarity}; /// Embedding dimensions - fixed at compile time for better performance @@ -119,7 +119,7 @@ fn create_distributed_embeddings( Some(distributed_vec) } -/// Performs NUMA-aware search using Fork Union's for_threads API for optimal colocation +/// Performs NUMA-aware search using ForkUnion's for_threads API for optimal colocation fn numa_aware_search( storage: &DistributedEmbeddings, query: &Embedding, diff --git a/scripts/test.c b/scripts/test.c new file mode 100644 index 0000000..6f83152 --- /dev/null +++ b/scripts/test.c @@ -0,0 +1,408 @@ +#include // `printf`, `fprintf` +#include // `EXIT_FAILURE`, `EXIT_SUCCESS` +#include // `atomic_size_t`, `atomic_fetch_add` +#include // `bool`, `true`, `false` +#include // `memset` + +#include + +/* Constants */ +static const size_t default_parallel_tasks_k = 10000; // 10K + +/* Test helpers */ +static bool test_try_spawn_zero(void) { + fu_pool_t *pool = fu_pool_new("test_zero"); + bool result = !fu_pool_spawn(pool, 0u, fu_caller_inclusive_k); + fu_pool_delete(pool); + return result; +} + +static bool test_try_spawn_success(void) { + fu_pool_t *pool = fu_pool_new("test_spawn"); + if (!pool) return false; + + size_t threads = fu_count_logical_cores(); + if (threads == 0) threads = 4; + + bool result = fu_pool_spawn(pool, threads, fu_caller_inclusive_k); + fu_pool_delete(pool); + return result; +} + +/* Context for for_threads test */ +struct for_threads_context { + atomic_bool *visited; +}; + +static void for_threads_callback(void *context_punned, size_t thread, size_t colocation) { + (void)colocation; + struct for_threads_context *context = (struct for_threads_context *)context_punned; + atomic_store(&context->visited[thread], true); +} + +static bool test_for_threads(void) { + fu_pool_t *pool = fu_pool_new("test_for_threads"); + if (!pool) return false; + + size_t threads = fu_count_logical_cores(); + if (threads == 0) threads = 4; + + if (!fu_pool_spawn(pool, threads, fu_caller_inclusive_k)) { + fu_pool_delete(pool); + return false; + } + + size_t threads_count = fu_pool_count_threads(pool); + atomic_bool *visited = calloc(threads_count, sizeof(atomic_bool)); + struct for_threads_context context = {.visited = visited}; + + fu_pool_for_threads(pool, for_threads_callback, &context); + + bool result = true; + for (size_t i = 0; i < threads_count; ++i) { + if (!atomic_load(&visited[i])) { + result = false; + break; + } + } + + free(visited); + fu_pool_delete(pool); + return result; +} + +/* Context for uncomfortable input size test */ +struct uncomfortable_context { + size_t input_size; + atomic_bool out_of_bounds; +}; + +static void uncomfortable_callback(void *context_punned, size_t task, size_t thread, size_t colocation) { + (void)thread; + (void)colocation; + struct uncomfortable_context *context = (struct uncomfortable_context *)context_punned; + if (task >= context->input_size) atomic_store(&context->out_of_bounds, true); +} + +static bool test_uncomfortable_input_size(void) { + fu_pool_t *pool = fu_pool_new("test_uncomfortable"); + if (!pool) return false; + + size_t threads = fu_count_logical_cores(); + if (threads == 0) threads = 4; + + if (!fu_pool_spawn(pool, threads, fu_caller_inclusive_k)) { + fu_pool_delete(pool); + return false; + } + + size_t threads_count = fu_pool_count_threads(pool); + size_t max_input_size = threads_count * 3; + + for (size_t input_size = 0; input_size <= max_input_size; ++input_size) { + struct uncomfortable_context context = {.input_size = input_size, .out_of_bounds = false}; + + fu_pool_for_n(pool, input_size, uncomfortable_callback, &context); + + if (atomic_load(&context.out_of_bounds)) { + fu_pool_delete(pool); + return false; + } + } + + fu_pool_delete(pool); + return true; +} + +/* Aligned visit structure for cache-line alignment */ +struct aligned_visit { + _Alignas(64) size_t task; +}; + +/* Comparator for qsort */ +static int compare_visits(const void *a, const void *b) { + const struct aligned_visit *va = (const struct aligned_visit *)a; + const struct aligned_visit *vb = (const struct aligned_visit *)b; + if (va->task < vb->task) return -1; + if (va->task > vb->task) return 1; + return 0; +} + +static bool contains_iota(struct aligned_visit *visited, size_t size) { + qsort(visited, size, sizeof(struct aligned_visit), compare_visits); + + for (size_t i = 0; i < size; ++i) + if (visited[i].task != i) return false; + return true; +} + +/* Context for for_n test */ +struct for_n_context { + atomic_size_t counter; + struct aligned_visit *visited; +}; + +static void for_n_callback(void *context_punned, size_t task, size_t thread, size_t colocation) { + (void)thread; + (void)colocation; + struct for_n_context *context = (struct for_n_context *)context_punned; + + size_t count_populated = atomic_fetch_add(&context->counter, 1); + context->visited[count_populated].task = task; +} + +static bool test_for_n(void) { + fu_pool_t *pool = fu_pool_new("test_for_n"); + if (!pool) return false; + + size_t threads = fu_count_logical_cores(); + if (threads == 0) threads = 4; + + if (!fu_pool_spawn(pool, threads, fu_caller_inclusive_k)) { + fu_pool_delete(pool); + return false; + } + + struct aligned_visit *visited = calloc(default_parallel_tasks_k, sizeof(struct aligned_visit)); + struct for_n_context context = {.counter = 0, .visited = visited}; + + fu_pool_for_n(pool, default_parallel_tasks_k, for_n_callback, &context); + + bool result = + (atomic_load(&context.counter) == default_parallel_tasks_k) && contains_iota(visited, default_parallel_tasks_k); + + if (result) { + // Test repeated calls + atomic_store(&context.counter, 0); + fu_pool_for_n(pool, default_parallel_tasks_k, for_n_callback, &context); + + result = (atomic_load(&context.counter) == default_parallel_tasks_k) && + contains_iota(visited, default_parallel_tasks_k); + } + + free(visited); + fu_pool_delete(pool); + return result; +} + +static bool test_for_n_dynamic(void) { + fu_pool_t *pool = fu_pool_new("test_for_n_dynamic"); + if (!pool) return false; + + size_t threads = fu_count_logical_cores(); + if (threads == 0) threads = 4; + + if (!fu_pool_spawn(pool, threads, fu_caller_inclusive_k)) { + fu_pool_delete(pool); + return false; + } + + struct aligned_visit *visited = calloc(default_parallel_tasks_k, sizeof(struct aligned_visit)); + struct for_n_context context = {.counter = 0, .visited = visited}; + + fu_pool_for_n_dynamic(pool, default_parallel_tasks_k, for_n_callback, &context); + + bool result = + (atomic_load(&context.counter) == default_parallel_tasks_k) && contains_iota(visited, default_parallel_tasks_k); + + if (result) { + // Test repeated calls + atomic_store(&context.counter, 0); + fu_pool_for_n_dynamic(pool, default_parallel_tasks_k, for_n_callback, &context); + + result = (atomic_load(&context.counter) == default_parallel_tasks_k) && + contains_iota(visited, default_parallel_tasks_k); + } + + free(visited); + fu_pool_delete(pool); + return result; +} + +static void oversubscribed_callback(void *context_punned, size_t task, size_t thread, size_t colocation) { + (void)thread; + (void)colocation; + struct for_n_context *context = (struct for_n_context *)context_punned; + + // Perform some weird amount of work, that is not very different between consecutive tasks + static _Thread_local volatile size_t some_local_work = 0; + for (size_t i = 0; i != task % 3; ++i) some_local_work = some_local_work + i * i; + + size_t count_populated = atomic_fetch_add(&context->counter, 1); + context->visited[count_populated].task = task; +} + +static bool test_oversubscribed_threads(void) { + const size_t oversubscription = 3; + + fu_pool_t *pool = fu_pool_new("test_oversubscribed"); + if (!pool) return false; + + size_t threads = fu_count_logical_cores(); + if (threads == 0) threads = 4; + + if (!fu_pool_spawn(pool, threads * oversubscription, fu_caller_inclusive_k)) { + fu_pool_delete(pool); + return false; + } + + struct aligned_visit *visited = calloc(default_parallel_tasks_k, sizeof(struct aligned_visit)); + struct for_n_context context = {.counter = 0, .visited = visited}; + + fu_pool_for_n_dynamic(pool, default_parallel_tasks_k, oversubscribed_callback, &context); + + bool result = + (atomic_load(&context.counter) == default_parallel_tasks_k) && contains_iota(visited, default_parallel_tasks_k); + + free(visited); + fu_pool_delete(pool); + return result; +} + +/* GCC nested functions extension test */ +#if defined(__GNUC__) && !defined(__clang__) + +static bool test_gcc_nested_functions(void) { + fu_pool_t *pool = fu_pool_new("test_gcc_nested"); + if (!pool) return false; + + size_t threads = fu_count_logical_cores(); + if (threads == 0) threads = 4; + + if (!fu_pool_spawn(pool, threads, fu_caller_inclusive_k)) { + fu_pool_delete(pool); + return false; + } + + atomic_size_t counter = 0; + size_t num_tasks = 100; + + /* GCC nested function - captures local variables */ + void nested_callback(void *context, size_t task, size_t thread, size_t colocation) { + (void)context; + (void)thread; + (void)colocation; + atomic_fetch_add(&counter, 1); + if (task % 20 == 0) printf(" GCC nested: Task %zu\n", task); + } + + fu_pool_for_n(pool, num_tasks, nested_callback, NULL); + + bool result = atomic_load(&counter) == num_tasks; + fu_pool_delete(pool); + return result; +} + +#endif // defined(__GNUC__) && !defined(__clang__) + +/* Clang blocks extension test */ +#if defined(__clang__) && defined(__BLOCKS__) + +#include + +typedef void (^task_block_t)(void *, size_t, size_t, size_t); + +struct block_wrapper { + task_block_t block; +}; + +static void block_callback_wrapper(void *context_punned, size_t task, size_t thread, size_t colocation) { + struct block_wrapper *wrapper = (struct block_wrapper *)context_punned; + wrapper->block(NULL, task, thread, colocation); +} + +static bool test_clang_blocks(void) { + fu_pool_t *pool = fu_pool_new("test_clang_blocks"); + if (!pool) return false; + + size_t threads = fu_count_logical_cores(); + if (threads == 0) threads = 4; + + if (!fu_pool_spawn(pool, threads, fu_caller_inclusive_k)) { + fu_pool_delete(pool); + return false; + } + + __block atomic_size_t counter = 0; + size_t num_tasks = 100; + + /* Clang block - captures local variables with __block */ + task_block_t my_block = ^(void *ctx, size_t task, size_t thread, size_t colocation) { + (void)ctx; + (void)thread; + (void)colocation; + atomic_fetch_add(&counter, 1); + if (task % 20 == 0) printf(" Clang block: Task %zu\n", task); + }; + + task_block_t heap_block = Block_copy(my_block); + struct block_wrapper wrapper = {.block = heap_block}; + + fu_pool_for_n(pool, num_tasks, block_callback_wrapper, &wrapper); + + Block_release(heap_block); + + bool result = atomic_load(&counter) == num_tasks; + fu_pool_delete(pool); + return result; +} + +#endif // defined(__clang__) && defined(__BLOCKS__) + +int main(void) { + printf("Welcome to the ForkUnion library test suite (C API)!\n"); + + char const *caps = fu_capabilities_string(); + if (!caps) { + fprintf(stderr, "Thread pool not supported on this platform\n"); + return EXIT_FAILURE; + } + + printf("Capabilities: %s\n", caps); + printf("Logical cores: %zu\n", fu_count_logical_cores()); + printf("NUMA nodes: %zu\n", fu_count_numa_nodes()); + printf("Colocations: %zu\n", fu_count_colocations()); + + printf("\nStarting unit tests...\n"); + + typedef bool (*test_func_t)(void); + struct { + char const *name; + test_func_t function; + } const unit_tests[] = { + {"`try_spawn` zero threads", test_try_spawn_zero}, + {"`try_spawn` normal", test_try_spawn_success}, + {"`for_threads` dispatch", test_for_threads}, + {"`for_n` for uncomfortable input size", test_uncomfortable_input_size}, + {"`for_n` static scheduling", test_for_n}, + {"`for_n_dynamic` dynamic scheduling", test_for_n_dynamic}, + {"`for_n_dynamic` oversubscribed threads", test_oversubscribed_threads}, +#if defined(__GNUC__) && !defined(__clang__) + {"GCC nested functions extension", test_gcc_nested_functions}, +#endif +#if defined(__clang__) && defined(__BLOCKS__) + {"Clang blocks extension", test_clang_blocks}, +#endif + }; + + size_t const total_unit_tests = sizeof(unit_tests) / sizeof(unit_tests[0]); + size_t failed_unit_tests = 0; + + for (size_t i = 0; i < total_unit_tests; ++i) { + printf("Running %s... ", unit_tests[i].name); + bool const ok = unit_tests[i].function(); + if (ok) printf("PASS\n"); + else + printf("FAIL\n"); + failed_unit_tests += !ok; + } + + if (failed_unit_tests > 0) { + fprintf(stderr, "%zu/%zu unit tests failed\n", failed_unit_tests, total_unit_tests); + return EXIT_FAILURE; + } + + printf("All %zu unit tests passed\n", total_unit_tests); + + return EXIT_SUCCESS; +} diff --git a/scripts/test.cpp b/scripts/test.cpp index 31145e8..31b688b 100644 --- a/scripts/test.cpp +++ b/scripts/test.cpp @@ -3,10 +3,10 @@ #include // `std::vector` #include // `std::sort` -#include +#include /* Namespaces, constants, and explicit type instantiations. */ -namespace fu = ashvardanian::fork_union; +namespace fu = ashvardanian::forkunion; using fu32_t = fu::basic_pool, fu::standard_yield_t, std::uint32_t>; using fu16_t = fu::basic_pool, fu::standard_yield_t, std::uint16_t>; @@ -95,11 +95,11 @@ struct make_pool_t { #if FU_ENABLE_NUMA static fu::numa_topology_t numa_topology; struct make_linux_colocated_pool_t { - fu::linux_colocated_pool_t construct() const noexcept { return fu::linux_colocated_pool_t("fork_union"); } + fu::linux_colocated_pool_t construct() const noexcept { return fu::linux_colocated_pool_t("forkunion"); } fu::numa_node_t scope(std::size_t = 0) const noexcept { return numa_topology.node(0); } }; struct make_linux_distributed_pool_t { - fu::linux_distributed_pool_t construct() const noexcept { return fu::linux_distributed_pool_t("fork_union"); } + fu::linux_distributed_pool_t construct() const noexcept { return fu::linux_distributed_pool_t("forkunion"); } fu::numa_topology_t const &scope(std::size_t = 0) const noexcept { return numa_topology; } }; #endif @@ -461,7 +461,7 @@ void log_numa_topology() noexcept { int main(void) { - std::printf("Welcome to the Fork Union library test suite!\n"); + std::printf("Welcome to the ForkUnion library test suite!\n"); log_numa_topology(); std::printf("Starting unit tests...\n"); diff --git a/zig/forkunion.zig b/zig/forkunion.zig new file mode 100644 index 0000000..a4fe29b --- /dev/null +++ b/zig/forkunion.zig @@ -0,0 +1,854 @@ +//! Low-latency OpenMP-style NUMA-aware cross-platform fine-grained parallelism library. +//! +//! ForkUnion provides a minimalistic cross-platform thread-pool implementation for fork-join +//! parallelism, avoiding dynamic memory allocations, exceptions, system calls, and heavy +//! Compare-And-Swap instructions on the hot path. +//! +//! Unlike std.Thread.Pool (which is a task queue for async work), ForkUnion is designed for +//! data parallelism and tight parallel loops - think OpenMP's `#pragma omp parallel for`. +//! +//! Basic usage: +//! ```zig +//! const fu = @import("forkunion"); +//! +//! var pool = try fu.Pool.init(4, .inclusive); +//! defer pool.deinit(); +//! +//! // Execute work on each thread (like OpenMP parallel) +//! pool.forThreads(struct { +//! fn work(thread_idx: usize, colocation_idx: usize) void { +//! std.debug.print("Thread {}\n", .{thread_idx}); +//! } +//! }.work, {}); +//! +//! // Distribute 1000 tasks across threads (like OpenMP parallel for) +//! var results = [_]i32{0} ** 1000; +//! pool.forN(1000, processTask, .{ .results = &results }); +//! ``` + +const std = @import("std"); +const builtin = @import("builtin"); + +// C ABI types +const c = struct { + extern fn fu_version_major() c_int; + extern fn fu_version_minor() c_int; + extern fn fu_version_patch() c_int; + extern fn fu_enabled_numa() c_int; + extern fn fu_capabilities_string() [*:0]const u8; + + extern fn fu_count_logical_cores() usize; + extern fn fu_count_colocations() usize; + extern fn fu_count_numa_nodes() usize; + extern fn fu_count_quality_levels() usize; + extern fn fu_volume_any_pages() usize; + extern fn fu_volume_any_pages_in(numa_node_index: usize) usize; + extern fn fu_volume_huge_pages_in(numa_node_index: usize) usize; + + extern fn fu_pool_new(name: ?[*:0]const u8) ?*anyopaque; + extern fn fu_pool_delete(pool: *anyopaque) void; + extern fn fu_pool_spawn(pool: *anyopaque, threads: usize, exclusivity: c_int) c_int; + extern fn fu_pool_terminate(pool: *anyopaque) void; + extern fn fu_pool_count_threads(pool: *anyopaque) usize; + extern fn fu_pool_count_colocations(pool: *anyopaque) usize; + extern fn fu_pool_count_threads_in(pool: *anyopaque, colocation_index: usize) usize; + extern fn fu_pool_locate_thread_in(pool: *anyopaque, global_thread_index: usize, colocation_index: usize) usize; + + extern fn fu_pool_for_threads( + pool: *anyopaque, + callback: *const fn (?*anyopaque, usize, usize) callconv(.c) void, + context: ?*anyopaque, + ) void; + extern fn fu_pool_for_n( + pool: *anyopaque, + n: usize, + callback: *const fn (?*anyopaque, usize, usize, usize) callconv(.c) void, + context: ?*anyopaque, + ) void; + extern fn fu_pool_for_n_dynamic( + pool: *anyopaque, + n: usize, + callback: *const fn (?*anyopaque, usize, usize, usize) callconv(.c) void, + context: ?*anyopaque, + ) void; + extern fn fu_pool_for_slices( + pool: *anyopaque, + n: usize, + callback: *const fn (?*anyopaque, usize, usize, usize, usize) callconv(.c) void, + context: ?*anyopaque, + ) void; + + extern fn fu_pool_unsafe_for_threads( + pool: *anyopaque, + callback: *const fn (?*anyopaque, usize, usize) callconv(.c) void, + context: ?*anyopaque, + ) void; + extern fn fu_pool_unsafe_join(pool: *anyopaque) void; + extern fn fu_pool_sleep(pool: *anyopaque, micros: usize) void; + + extern fn fu_allocate_at_least( + numa_node_index: usize, + minimum_bytes: usize, + allocated_bytes: *usize, + bytes_per_page: *usize, + ) ?*anyopaque; + extern fn fu_allocate(numa_node_index: usize, bytes: usize) ?*anyopaque; + extern fn fu_free(numa_node_index: usize, pointer: *anyopaque, bytes: usize) void; +}; + +/// Errors that can occur during thread pool operations +pub const Error = error{ + /// Failed to create thread pool + CreationFailed, + /// Failed to spawn worker threads + SpawnFailed, + /// Platform not supported + UnsupportedPlatform, +}; + +/// Defines whether the calling thread participates in task execution +pub const CallerExclusivity = enum(c_int) { + /// Calling thread participates in workload (spawns N-1 workers) + inclusive = 0, + /// Calling thread only coordinates (spawns N workers) + exclusive = 1, +}; + +/// A "prong" - metadata about a task's execution context +pub const Prong = struct { + /// The logical index of the task being processed + task_index: usize, + /// The physical thread executing this task + thread_index: usize, + /// The colocation group (NUMA node + QoS level) + colocation_index: usize, +}; + +/// Returns the library version as a struct +pub fn version() struct { major: u32, minor: u32, patch: u32 } { + return .{ + .major = @intCast(c.fu_version_major()), + .minor = @intCast(c.fu_version_minor()), + .patch = @intCast(c.fu_version_patch()), + }; +} + +/// Returns true if NUMA support was compiled into the library +pub fn numaEnabled() bool { + return c.fu_enabled_numa() != 0; +} + +/// Returns a string describing available platform capabilities +pub fn capabilitiesString() [*:0]const u8 { + return c.fu_capabilities_string(); +} + +/// Returns the number of logical CPU cores available +pub fn countLogicalCores() usize { + return c.fu_count_logical_cores(); +} + +/// Returns the number of NUMA nodes available +pub fn countNumaNodes() usize { + return c.fu_count_numa_nodes(); +} + +/// Returns the number of distinct thread colocations +pub fn countColocations() usize { + return c.fu_count_colocations(); +} + +/// Returns the number of distinct Quality-of-Service levels +pub fn countQualityLevels() usize { + return c.fu_count_quality_levels(); +} + +/// Returns total volume of pages available across all NUMA nodes +pub fn volumeAnyPages() usize { + return c.fu_volume_any_pages(); +} + +/// Returns volume of pages available on a specific NUMA node +pub fn volumeAnyPagesIn(numa_node_index: usize) usize { + return c.fu_volume_any_pages_in(numa_node_index); +} + +/// Returns volume of huge pages available on a specific NUMA node +pub fn volumeHugePagesIn(numa_node_index: usize) usize { + return c.fu_volume_huge_pages_in(numa_node_index); +} + +/// NUMA-aware memory allocation result +pub const NumaAllocation = struct { + ptr: [*]u8, + allocated_bytes: usize, + bytes_per_page: usize, + numa_node: usize, + + /// Returns the allocated memory as a slice + pub fn asSlice(self: NumaAllocation) []u8 { + return self.ptr[0..self.allocated_bytes]; + } + + /// Frees the NUMA allocation + pub fn free(self: NumaAllocation) void { + c.fu_free(self.numa_node, @ptrCast(self.ptr), self.allocated_bytes); + } +}; + +/// Allocates memory on a specific NUMA node with optimal page size +pub fn allocateAtLeast(numa_node_index: usize, minimum_bytes: usize) ?NumaAllocation { + var allocated_bytes: usize = undefined; + var bytes_per_page: usize = undefined; + + const ptr = c.fu_allocate_at_least( + numa_node_index, + minimum_bytes, + &allocated_bytes, + &bytes_per_page, + ) orelse return null; + + return .{ + .ptr = @ptrCast(@alignCast(ptr)), + .allocated_bytes = allocated_bytes, + .bytes_per_page = bytes_per_page, + .numa_node = numa_node_index, + }; +} + +/// Allocates exactly the requested bytes on a specific NUMA node +pub fn allocate(numa_node_index: usize, bytes: usize) ?[*]u8 { + const ptr = c.fu_allocate(numa_node_index, bytes) orelse return null; + return @ptrCast(@alignCast(ptr)); +} + +/// NUMA-aware allocator compatible with Zig's allocator interface. +pub const NumaAllocator = struct { + node_index: usize, + + const Self = @This(); + const Allocator = std.mem.Allocator; + + const Header = packed struct { + base_addr: usize, + allocated_bytes: usize, + }; + + const vtable = Allocator.VTable{ + .alloc = alloc, + .resize = resize, + .remap = remap, + .free = free, + }; + + pub fn init(node_index: usize) Self { + return .{ .node_index = node_index }; + } + + pub fn allocator(self: *Self) Allocator { + return .{ .ptr = self, .vtable = &vtable }; + } + + fn alloc(ctx: *anyopaque, len: usize, alignment: std.mem.Alignment, ret_addr: usize) ?[*]u8 { + _ = ret_addr; + const self: *Self = @ptrCast(@alignCast(ctx)); + const effective_len = if (len == 0) 1 else len; + const slice = self.allocSlice(effective_len, alignment) orelse return null; + return slice.ptr; + } + + fn resize( + ctx: *anyopaque, + buf: []u8, + alignment: std.mem.Alignment, + new_len: usize, + ret_addr: usize, + ) bool { + _ = ret_addr; + const self: *Self = @ptrCast(@alignCast(ctx)); + return self.resizeInPlace(buf, alignment, new_len); + } + + fn remap( + ctx: *anyopaque, + buf: []u8, + alignment: std.mem.Alignment, + new_len: usize, + ret_addr: usize, + ) ?[*]u8 { + _ = ret_addr; + const self: *Self = @ptrCast(@alignCast(ctx)); + const result = self.remapSlice(buf, alignment, new_len) orelse return null; + return result.ptr; + } + + fn free(ctx: *anyopaque, buf: []u8, alignment: std.mem.Alignment, ret_addr: usize) void { + _ = alignment; + _ = ret_addr; + const self: *Self = @ptrCast(@alignCast(ctx)); + self.freeSlice(buf); + } + + fn allocSlice(self: *Self, len: usize, alignment: std.mem.Alignment) ?[]u8 { + const header_size = @sizeOf(Header); + const alignment_bytes = alignment.toByteUnits(); + const with_header = std.math.add(usize, len, header_size) catch return null; + const request_bytes = std.math.add(usize, with_header, alignment_bytes) catch return null; + + var allocated_bytes: usize = undefined; + var bytes_per_page: usize = undefined; + const raw_ptr = c.fu_allocate_at_least( + self.node_index, + request_bytes, + &allocated_bytes, + &bytes_per_page, + ) orelse return null; + + const base_addr = @intFromPtr(raw_ptr); + const data_addr = alignment.forward(base_addr + header_size); + if (data_addr + len > base_addr + allocated_bytes) { + c.fu_free(self.node_index, raw_ptr, allocated_bytes); + return null; + } + + const header_ptr = @as(*Header, @ptrFromInt(data_addr - header_size)); + header_ptr.* = .{ + .base_addr = base_addr, + .allocated_bytes = allocated_bytes, + }; + + const data_ptr = @as([*]u8, @ptrFromInt(data_addr)); + return data_ptr[0..len]; + } + + fn resizeInPlace(self: *Self, buf: []u8, alignment: std.mem.Alignment, new_len: usize) bool { + _ = self; + _ = alignment; + if (buf.len == 0) return false; + if (new_len == 0) return false; + if (new_len <= buf.len) return true; + return false; + } + + fn remapSlice(self: *Self, buf: []u8, alignment: std.mem.Alignment, new_len: usize) ?[]u8 { + if (buf.len == 0) return null; + if (new_len == 0) { + self.freeSlice(buf); + return buf[0..0]; + } + if (new_len <= buf.len) return buf[0..new_len]; + + const new_slice = self.allocSlice(new_len, alignment) orelse return null; + @memcpy(new_slice[0..buf.len], buf); + self.freeSlice(buf); + return new_slice; + } + + fn freeSlice(self: *Self, buf: []u8) void { + if (buf.len == 0) return; + const header_ptr = @as(*Header, @ptrFromInt(@intFromPtr(buf.ptr) - @sizeOf(Header))); + const header = header_ptr.*; + const base_ptr = @as(*anyopaque, @ptrFromInt(header.base_addr)); + c.fu_free(self.node_index, base_ptr, header.allocated_bytes); + } +}; + +/// Thread pool for fork-join parallelism +pub const Pool = struct { + handle: *anyopaque, + + /// Creates a new thread pool + pub fn init(thread_count: usize, exclusivity: CallerExclusivity) Error!Pool { + return initNamed(null, thread_count, exclusivity); + } + + /// Creates a new named thread pool + pub fn initNamed( + name: ?[]const u8, + thread_count: usize, + exclusivity: CallerExclusivity, + ) Error!Pool { + // Convert name to null-terminated string if provided + // SAFETY: C library copies name into internal buffer immediately + var name_buf: [16:0]u8 = undefined; + const name_z: ?[*:0]const u8 = if (name) |n| + std.fmt.bufPrintZ(&name_buf, "{s}", .{n[0..@min(n.len, 15)]}) catch unreachable + else + null; + + const handle = c.fu_pool_new(name_z) orelse return Error.CreationFailed; + errdefer c.fu_pool_delete(handle); + + // C++ validates threads > 0 and returns false if invalid + const success = c.fu_pool_spawn(handle, thread_count, @intFromEnum(exclusivity)); + if (success == 0) return Error.SpawnFailed; + + return .{ .handle = handle }; + } + + /// Destroys the thread pool + pub fn deinit(self: Pool) void { + c.fu_pool_delete(self.handle); + } + + /// Returns the number of threads in the pool + pub fn threads(self: *const Pool) usize { + return c.fu_pool_count_threads(self.handle); + } + + /// Returns the number of colocations in the pool + pub fn colocations(self: *const Pool) usize { + return c.fu_pool_count_colocations(self.handle); + } + + /// Returns the number of threads in a specific colocation + pub fn countThreadsIn(self: *const Pool, colocation_index: usize) usize { + return c.fu_pool_count_threads_in(self.handle, colocation_index); + } + + /// Converts global thread index to local index within colocation + pub fn locateThreadIn(self: *const Pool, global_thread_index: usize, colocation_index: usize) usize { + return c.fu_pool_locate_thread_in(self.handle, global_thread_index, colocation_index); + } + + /// Terminates all worker threads (pool can be respawned) + pub fn terminate(self: *const Pool) void { + c.fu_pool_terminate(self.handle); + } + + /// Puts worker threads into power-saving sleep state + pub fn sleep(self: *const Pool, microseconds: usize) void { + c.fu_pool_sleep(self.handle, microseconds); + } + + /// Executes a callback on all threads (blocking) + /// Note: context parameter exists for API uniformity but is not passed to the callback + pub fn forThreads( + self: *const Pool, + comptime func: fn (usize, usize) void, + context: anytype, + ) void { + _ = context; // Not used - kept for API consistency with forN/forNDynamic/forSlices + const Wrapper = struct { + fn callback(_: ?*anyopaque, thread_idx: usize, colocation_idx: usize) callconv(.c) void { + func(thread_idx, colocation_idx); + } + }; + + c.fu_pool_for_threads(self.handle, Wrapper.callback, null); + } + + /// Distributes N tasks across threads with static scheduling (blocking) + /// + /// The callback function signature must match the context type: + /// - If context is `void`: `fn(Prong) void` + /// - If context is type `T`: `fn(Prong, T) void` + pub fn forN( + self: *const Pool, + n: usize, + comptime func: anytype, + context: anytype, + ) void { + const Context = @TypeOf(context); + + // Validate function signature at compile time + const expected_type = if (Context == void) + fn (Prong) void + else + fn (Prong, Context) void; + + if (@TypeOf(func) != expected_type) { + @compileError("Function signature must be: " ++ @typeName(expected_type)); + } + + if (Context == void) { + // Stateless path - no context + const Wrapper = struct { + fn callback( + _: ?*anyopaque, + task_idx: usize, + thread_idx: usize, + colocation_idx: usize, + ) callconv(.c) void { + const prong = Prong{ + .task_index = task_idx, + .thread_index = thread_idx, + .colocation_index = colocation_idx, + }; + func(prong, {}); + } + }; + c.fu_pool_for_n(self.handle, n, Wrapper.callback, null); + } else { + // Stateful path - pass context + const Wrapper = struct { + fn callback( + ctx: ?*anyopaque, + task_idx: usize, + thread_idx: usize, + colocation_idx: usize, + ) callconv(.c) void { + const prong = Prong{ + .task_index = task_idx, + .thread_index = thread_idx, + .colocation_index = colocation_idx, + }; + // SAFETY: Context pointer valid for duration of blocking call + const typed_ctx: *const Context = @ptrCast(@alignCast(ctx)); + func(prong, typed_ctx.*); + } + }; + c.fu_pool_for_n(self.handle, n, Wrapper.callback, @ptrCast(@constCast(&context))); + } + } + + /// Distributes N tasks with dynamic work-stealing (blocking) + /// + /// The callback function signature must match the context type: + /// - If context is `void`: `fn(Prong) void` + /// - If context is type `T`: `fn(Prong, T) void` + pub fn forNDynamic( + self: *const Pool, + n: usize, + comptime func: anytype, + context: anytype, + ) void { + const Context = @TypeOf(context); + + // Validate function signature at compile time + const expected_type = if (Context == void) + fn (Prong) void + else + fn (Prong, Context) void; + + if (@TypeOf(func) != expected_type) { + @compileError("Function signature must be: " ++ @typeName(expected_type)); + } + + if (Context == void) { + // Stateless path - no context + const Wrapper = struct { + fn callback( + _: ?*anyopaque, + task_idx: usize, + thread_idx: usize, + colocation_idx: usize, + ) callconv(.c) void { + const prong = Prong{ + .task_index = task_idx, + .thread_index = thread_idx, + .colocation_index = colocation_idx, + }; + func(prong, {}); + } + }; + c.fu_pool_for_n_dynamic(self.handle, n, Wrapper.callback, null); + } else { + // Stateful path - pass context + const Wrapper = struct { + fn callback( + ctx: ?*anyopaque, + task_idx: usize, + thread_idx: usize, + colocation_idx: usize, + ) callconv(.c) void { + const prong = Prong{ + .task_index = task_idx, + .thread_index = thread_idx, + .colocation_index = colocation_idx, + }; + // SAFETY: Context pointer valid for duration of blocking call + const typed_ctx: *const Context = @ptrCast(@alignCast(ctx)); + func(prong, typed_ctx.*); + } + }; + c.fu_pool_for_n_dynamic(self.handle, n, Wrapper.callback, @ptrCast(@constCast(&context))); + } + } + + /// Distributes N tasks as slices (blocking) + /// + /// The callback function signature must match the context type: + /// - If context is `void`: `fn(Prong, usize) void` + /// - If context is type `T`: `fn(Prong, usize, T) void` + /// + /// The second parameter is the slice count for this chunk. + pub fn forSlices( + self: *const Pool, + n: usize, + comptime func: anytype, + context: anytype, + ) void { + const Context = @TypeOf(context); + + // Validate function signature at compile time + const expected_type = if (Context == void) + fn (Prong, usize) void + else + fn (Prong, usize, Context) void; + + if (@TypeOf(func) != expected_type) { + @compileError("Function signature must be: " ++ @typeName(expected_type)); + } + + if (Context == void) { + // Stateless path - no context + const Wrapper = struct { + fn callback( + _: ?*anyopaque, + first_idx: usize, + count: usize, + thread_idx: usize, + colocation_idx: usize, + ) callconv(.c) void { + const prong = Prong{ + .task_index = first_idx, + .thread_index = thread_idx, + .colocation_index = colocation_idx, + }; + func(prong, count); + } + }; + c.fu_pool_for_slices(self.handle, n, Wrapper.callback, null); + } else { + // Stateful path - pass context + const Wrapper = struct { + fn callback( + ctx: ?*anyopaque, + first_idx: usize, + count: usize, + thread_idx: usize, + colocation_idx: usize, + ) callconv(.c) void { + const prong = Prong{ + .task_index = first_idx, + .thread_index = thread_idx, + .colocation_index = colocation_idx, + }; + // SAFETY: Context pointer valid for duration of blocking call + const typed_ctx: *const Context = @ptrCast(@alignCast(ctx)); + func(prong, count, typed_ctx.*); + } + }; + c.fu_pool_for_slices(self.handle, n, Wrapper.callback, @ptrCast(@constCast(&context))); + } + } + + /// Executes callback on all threads without blocking (unsafe) + pub fn unsafeForThreads( + self: *const Pool, + comptime func: fn (usize, usize) void, + context: anytype, + ) void { + _ = context; // Not used - kept for API consistency + const Wrapper = struct { + fn callback(_: ?*anyopaque, thread_idx: usize, colocation_idx: usize) callconv(.c) void { + func(thread_idx, colocation_idx); + } + }; + + c.fu_pool_unsafe_for_threads(self.handle, Wrapper.callback, null); + } + + /// Blocks until current parallel operation completes (unsafe) + pub fn unsafeJoin(self: *const Pool) void { + c.fu_pool_unsafe_join(self.handle); + } +}; + +// ============================================================================ +// Tests +// ============================================================================ + +test "version info" { + std.debug.print("Running test: version info\n", .{}); + const v = version(); + try std.testing.expect(v.major >= 0); + try std.testing.expect(v.minor >= 0); +} + +test "system capabilities" { + std.debug.print("Running test: system capabilities\n", .{}); + const caps = capabilitiesString(); + try std.testing.expect(std.mem.len(caps) > 0); +} + +test "system metadata" { + std.debug.print("Running test: system metadata\n", .{}); + const cores = countLogicalCores(); + try std.testing.expect(cores > 0); + + const numa = countNumaNodes(); + try std.testing.expect(numa >= 0); + + const colocs = countColocations(); + try std.testing.expect(colocs > 0); +} + +test "pool creation and destruction" { + std.debug.print("Running test: pool creation and destruction\n", .{}); + var pool = try Pool.init(2, .inclusive); + defer pool.deinit(); + + try std.testing.expectEqual(@as(usize, 2), pool.threads()); +} + +test "named pool creation" { + std.debug.print("Running test: named pool creation\n", .{}); + var pool = try Pool.initNamed(null, 2, .inclusive); + defer pool.deinit(); + + try std.testing.expectEqual(@as(usize, 2), pool.threads()); +} + +test "for_threads execution" { + std.debug.print("Running test: for_threads execution\n", .{}); + var pool = try Pool.init(4, .inclusive); + defer pool.deinit(); + + const State = struct { + var visited: [4]std.atomic.Value(bool) = [_]std.atomic.Value(bool){ + std.atomic.Value(bool).init(false), + std.atomic.Value(bool).init(false), + std.atomic.Value(bool).init(false), + std.atomic.Value(bool).init(false), + }; + + fn worker(thread_idx: usize, colocation_idx: usize) void { + _ = colocation_idx; + if (thread_idx < 4) { + visited[thread_idx].store(true, .release); + } + } + }; + + pool.forThreads(State.worker, {}); + + // Verify all threads executed + for (0..4) |i| { + try std.testing.expect(State.visited[i].load(.acquire)); + } +} + +test "for_n static scheduling" { + std.debug.print("Running test: for_n static scheduling\n", .{}); + var pool = try Pool.init(4, .inclusive); + defer pool.deinit(); + + var visited = [_]std.atomic.Value(bool){std.atomic.Value(bool).init(false)} ** 100; + + const Context = struct { + visited_ptr: *[100]std.atomic.Value(bool), + }; + + pool.forN(100, struct { + fn worker(prong: Prong, ctx: Context) void { + ctx.visited_ptr[prong.task_index].store(true, .release); + } + }.worker, Context{ .visited_ptr = &visited }); + + // Verify all tasks executed + for (0..100) |i| { + try std.testing.expect(visited[i].load(.acquire)); + } +} + +test "for_n_dynamic work stealing" { + std.debug.print("Running test: for_n_dynamic work stealing\n", .{}); + var pool = try Pool.init(4, .inclusive); + defer pool.deinit(); + + var counter = std.atomic.Value(usize).init(0); + + const Context = struct { + counter_ptr: *std.atomic.Value(usize), + }; + + pool.forNDynamic(100, struct { + fn worker(prong: Prong, ctx: Context) void { + _ = prong; + _ = ctx.counter_ptr.fetchAdd(1, .monotonic); + } + }.worker, Context{ .counter_ptr = &counter }); + + try std.testing.expectEqual(@as(usize, 100), counter.load(.acquire)); +} + +test "for_slices execution" { + std.debug.print("Running test: for_slices execution\n", .{}); + var pool = try Pool.init(4, .inclusive); + defer pool.deinit(); + + var data = [_]i32{0} ** 1000; + var total = std.atomic.Value(usize).init(0); + + const Context = struct { + data_ptr: *[1000]i32, + total_ptr: *std.atomic.Value(usize), + }; + + pool.forSlices(1000, struct { + fn worker(prong: Prong, count: usize, ctx: Context) void { + var local_sum: usize = 0; + for (0..count) |i| { + const idx = prong.task_index + i; + ctx.data_ptr[idx] = @intCast(idx); + local_sum += 1; + } + _ = ctx.total_ptr.fetchAdd(local_sum, .monotonic); + } + }.worker, Context{ .data_ptr = &data, .total_ptr = &total }); + + // Verify all elements were processed + try std.testing.expectEqual(@as(usize, 1000), total.load(.acquire)); + for (0..1000) |i| { + try std.testing.expectEqual(@as(i32, @intCast(i)), data[i]); + } +} + +test "NUMA allocation" { + std.debug.print("Running test: NUMA allocation\n", .{}); + if (!numaEnabled()) return error.SkipZigTest; + + const allocation = allocateAtLeast(0, 1024) orelse return error.SkipZigTest; + defer allocation.free(); + + try std.testing.expect(allocation.allocated_bytes >= 1024); + try std.testing.expectEqual(@as(usize, 0), allocation.numa_node); + + // Write to memory to ensure it's usable + const slice = allocation.asSlice(); + for (0..@min(1024, slice.len)) |i| { + slice[i] = @intCast(i & 0xFF); + } +} + +test "NUMA allocator integrates with std collections" { + std.debug.print("Running test: NUMA allocator integrates with std collections\n", .{}); + if (!numaEnabled()) return error.SkipZigTest; + + var numa_alloc = NumaAllocator.init(0); + const allocator = numa_alloc.allocator(); + + var list = try std.ArrayList(u64).initCapacity(allocator, 0); + defer list.deinit(allocator); + try list.appendSlice(allocator, &[_]u64{ 1, 2, 3, 4, 5 }); + try std.testing.expectEqual(@as(usize, 5), list.items.len); + try std.testing.expectEqual(@as(u64, 3), list.items[2]); + + var map = std.AutoHashMap(u32, u32).init(allocator); + defer map.deinit(); + try map.put(10, 100); + try map.put(20, 200); + try map.put(30, 300); + try std.testing.expectEqual(@as(usize, 3), map.count()); + try std.testing.expectEqual(@as(u32, 200), map.get(20).?); + + var buf = try allocator.alloc(u8, 128); + defer allocator.free(buf); + @memset(buf, 0xAB); + + buf = try allocator.realloc(buf, 512); + try std.testing.expectEqual(@as(usize, 512), buf.len); + try std.testing.expectEqual(@as(u8, 0xAB), buf[0]); +}