Multi-threaded reductions

.github/workflows/build.yml

@@ -12,6 +12,8 @@ jobs:
   build_wheels:
     name: Build and test on ${{ matrix.os }}${{ matrix.numpy-version && format(' (numpy {0})', matrix.numpy-version) || '' }}
     runs-on: ${{ matrix.os }}
+    env:
+      CMAKE_GENERATOR: Ninja
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
@@ -30,15 +32,65 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
+    - name: Install sccache (Windows)
+      if: runner.os == 'Windows'
+      run: choco install sccache --yes
+
+    - name: Cache sccache (Windows)
+      if: runner.os == 'Windows'
+      uses: actions/cache@v4
+      with:
+        path: C:\Users\runneradmin\AppData\Local\sccache
+        key: sccache-${{ runner.os }}-${{ github.sha }}
+        restore-keys: |
+          sccache-${{ runner.os }}-
+
+    - name: Cache pip (Windows)
+      if: runner.os == 'Windows'
+      uses: actions/cache@v4
+      with:
+        path: C:\Users\runneradmin\AppData\Local\pip\Cache
+        key: pip-${{ runner.os }}-${{ hashFiles('pyproject.toml') }}
+        restore-keys: |
+          pip-${{ runner.os }}-
+
     - name: Install Ninja
       uses: seanmiddleditch/gha-setup-ninja@master
 
+    - name: Add LLVM to PATH (Windows)
+      if: runner.os == 'Windows'
+      run: echo "C:\\Program Files\\LLVM\\bin" >> $env:GITHUB_PATH
+
     - name: Install specific numpy version
       if: matrix.numpy-version
       run: pip install "numpy==${{ matrix.numpy-version }}.*"
 
-    - name: Build
+    - name: Build (Windows)
+      if: runner.os == 'Windows'
+      id: build_windows
+      run: pip install -e .[test]
+      env:
+        CMAKE_C_COMPILER_LAUNCHER: sccache
+        CMAKE_CXX_COMPILER_LAUNCHER: sccache
+        SCCACHE_DIR: C:\Users\runneradmin\AppData\Local\sccache
+        CC: clang-cl
+        CXX: clang-cl
+        CMAKE_BUILD_PARALLEL_LEVEL: 8
+        SKBUILD_PARALLEL_LEVEL: 8
+
+    - name: Build (non-Windows)
+      if: runner.os != 'Windows'
+      id: build_non_windows
       run: pip install -e .[test]
 
-    - name: Test
+    - name: Test (Windows)
+      if: runner.os == 'Windows'
+      run: python -m pytest -m "not heavy and (network or not network)"
+#      env:
+#        BLOSC_NTHREADS: "1"
+#        NUMEXPR_NUM_THREADS: "1"
+#        OMP_NUM_THREADS: "1"
+
+    - name: Test (non-Windows)
+      if: runner.os != 'Windows'
       run: python -m pytest -m "not heavy and (network or not network)"

.github/workflows/cibuildwheels.yml

@@ -17,6 +17,11 @@ env:
   # Skip PyPy wheels for now (numexpr needs some adjustments first)
   # musllinux takes too long to build, and it's not worth it for now
   CIBW_SKIP: "pp* *musllinux* *-win32"
+  # Use explicit generator/compiler env vars; CMAKE_ARGS with spaces is not split on Windows.
+  CIBW_ENVIRONMENT_WINDOWS: >-
+    CMAKE_GENERATOR=Ninja
+    CC=clang-cl
+    CXX=clang-cl
 
 jobs:
 
@@ -77,6 +82,10 @@ jobs:
         id: ninja
         uses: turtlesec-no/get-ninja@main
 
+      - name: Add LLVM to PATH (Windows)
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: echo "C:\\Program Files\\LLVM\\bin" >> $env:GITHUB_PATH
+
       - name: Install MSVC amd64
         uses: ilammy/msvc-dev-cmd@v1
         with:

.github/workflows/wasm.yml

@@ -21,6 +21,7 @@ jobs:
     env:
       CIBW_BUILD: ${{ matrix.cibw_build }}
       CMAKE_ARGS: "-DWITH_OPTIM=OFF"
+      DEACTIVATE_OPENZL: "1"
       CIBW_TEST_COMMAND: "pytest {project}/tests/ndarray/test_reductions.py"
     strategy:
       matrix:

CMakeLists.txt

@@ -1,5 +1,22 @@
 cmake_minimum_required(VERSION 3.15.0)
+
+if(WIN32)
+    cmake_policy(SET CMP0091 NEW)
+    set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>DLL" CACHE STRING "" FORCE)
+endif()
+
+
+if(WIN32 AND CMAKE_GENERATOR MATCHES "Visual Studio")
+    if(NOT DEFINED CMAKE_GENERATOR_TOOLSET)
+        set(CMAKE_GENERATOR_TOOLSET "ClangCL" CACHE STRING "Use ClangCL toolset for C99/C11 support on Windows." FORCE)
+    endif()
+endif()
+
 project(python-blosc2)
+
+if(WIN32 AND NOT CMAKE_C_COMPILER_ID STREQUAL "Clang")
+    message(FATAL_ERROR "Windows builds require clang-cl. Set CC/CXX to clang-cl or configure CMake with -T ClangCL.")
+endif()
 # Specifying Python version below is tricky, but if you don't specify the minimum version here,
 # it would not consider python3 when looking for the executable. This is problematic since Fedora
 # does not include a python symbolic link to python3.
@@ -23,11 +40,55 @@ add_custom_command(
           "${CMAKE_CURRENT_SOURCE_DIR}/src/blosc2/blosc2_ext.pyx" --output-file blosc2_ext.c
   DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/blosc2/blosc2_ext.pyx"
   VERBATIM)
+
 # ...and add it to the target
 Python_add_library(blosc2_ext MODULE blosc2_ext.c WITH_SOABI)
+
 # We need to link against NumPy
 target_link_libraries(blosc2_ext PRIVATE Python::NumPy)
 
+# Fetch and build miniexpr library
+include(FetchContent)
+
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+set(MINIEXPR_BUILD_SHARED OFF CACHE BOOL "Build miniexpr shared library" FORCE)
+set(MINIEXPR_BUILD_TESTS OFF CACHE BOOL "Build miniexpr tests" FORCE)
+set(MINIEXPR_BUILD_EXAMPLES OFF CACHE BOOL "Build miniexpr examples" FORCE)
+set(MINIEXPR_BUILD_BENCH OFF CACHE BOOL "Build miniexpr benchmarks" FORCE)
+
+FetchContent_Declare(miniexpr
+    GIT_REPOSITORY https://github.com/Blosc/miniexpr.git
+    GIT_TAG sleef  # latest SIMD additions
+    # In case you want to use a local copy of miniexpr for development, uncomment the line below
+    # SOURCE_DIR "/Users/faltet/blosc/miniexpr"
+)
+FetchContent_MakeAvailable(miniexpr)
+
+# Link against miniexpr static library
+target_link_libraries(blosc2_ext PRIVATE miniexpr_static)
+
+target_compile_features(blosc2_ext PRIVATE c_std_11)
+if(WIN32 AND CMAKE_C_COMPILER_ID STREQUAL "Clang")
+    execute_process(
+        COMMAND "${CMAKE_C_COMPILER}" -print-resource-dir
+        OUTPUT_VARIABLE _clang_resource_dir
+        OUTPUT_STRIP_TRAILING_WHITESPACE
+        ERROR_QUIET
+    )
+    if(_clang_resource_dir)
+        if(CMAKE_SIZEOF_VOID_P EQUAL 8)
+            set(_clang_builtins "${_clang_resource_dir}/lib/windows/clang_rt.builtins-x86_64.lib")
+        else()
+            set(_clang_builtins "${_clang_resource_dir}/lib/windows/clang_rt.builtins-i386.lib")
+        endif()
+        if(EXISTS "${_clang_builtins}")
+            target_link_libraries(blosc2_ext PRIVATE "${_clang_builtins}")
+        endif()
+        unset(_clang_builtins)
+    endif()
+    unset(_clang_resource_dir)
+endif()
+
 if(DEFINED ENV{USE_SYSTEM_BLOSC2})
     set(USE_SYSTEM_BLOSC2 ON)
 endif()
@@ -44,13 +105,20 @@ else()
     set(BUILD_EXAMPLES OFF CACHE BOOL "Build C-Blosc2 examples")
     set(BUILD_BENCHMARKS OFF CACHE BOOL "Build C-Blosc2 benchmarks")
     set(BUILD_FUZZERS OFF CACHE BOOL "Build C-Blosc2 fuzzers")
+    if(DEFINED ENV{DEACTIVATE_OPENZL})
+        set(DEACTIVATE_OPENZL ON CACHE BOOL "Do not include support for the OpenZL library.")
+    endif()
     set(CMAKE_POSITION_INDEPENDENT_CODE ON)
     # we want the binaries of the C-Blosc2 library to go into the wheels
     set(BLOSC_INSTALL ON)
     include(FetchContent)
     FetchContent_Declare(blosc2
         GIT_REPOSITORY https://github.com/Blosc/c-blosc2
-        GIT_TAG 5a2b0ed9c4d801230c118fbc5811817055b5a3f5  # v2.22.0
+        # GIT_TAG 9d250c2201f6e385c56a372b08037f7debc6fa1b  # openzl (disposable output)
+        GIT_TAG add_openzl
+        GIT_SHALLOW TRUE  # fetch only the latest commit (only works with a branch in GIT_TAG)
+        # in case you want to use a local copy of c-blosc2 for development, uncomment the line below
+        # SOURCE_DIR "/Users/faltet/blosc/c-blosc2"
     )
     FetchContent_MakeAvailable(blosc2)
     include_directories("${blosc2_SOURCE_DIR}/include")

README.rst

@@ -53,6 +53,17 @@ Conda users can install from conda-forge:
 
     conda install -c conda-forge python-blosc2
 
+Windows note
+============
+
+When building from source on Windows, clang-cl is required (OpenZL depends on C11 support).
+Make sure LLVM is on PATH and use the Ninja generator, for example::
+
+    CMAKE_GENERATOR=Ninja
+    CC=clang-cl
+    CXX=clang-cl
+    pip install -e .
+
 Documentation
 =============
 

README_DEVELOPERS.md

@@ -22,6 +22,16 @@ You are done!
     pip install .   # add -e for editable mode
 ```
 
+On Windows, clang-cl is required (OpenZL depends on C11 support). Make sure LLVM
+is on PATH and build with Ninja, for example:
+
+```bash
+CMAKE_GENERATOR=Ninja \
+CC=clang-cl \
+CXX=clang-cl \
+pip install -e .
+```
+
 There are situations where you may want to build the C-Blosc2 library separately, for example, when debugging issues in the C library. In that case, let's assume you have the C-Blosc2 library installed in `/usr/local`:
 
 ```bash
@@ -38,6 +48,37 @@ LD_LIBRARY_PATH=/usr/local/lib pytest
 
 That's it! You can now proceed to the testing section.
 
+### Speeding up local builds (sccache + Ninja)
+
+If you do frequent local rebuilds, sccache can significantly speed up C/C++ rebuilds.
+
+```bash
+brew install sccache ninja
+```
+
+Then run:
+
+```bash
+CMAKE_GENERATOR=Ninja \
+CMAKE_C_COMPILER=clang \
+CMAKE_CXX_COMPILER=clang++ \
+CMAKE_C_COMPILER_LAUNCHER=sccache \
+CMAKE_CXX_COMPILER_LAUNCHER=sccache \
+CMAKE_BUILD_PARALLEL_LEVEL=8 \
+SKBUILD_PARALLEL_LEVEL=8 \
+SKBUILD_BUILD_DIR=build \
+pip install -e .
+```
+
+Using `SKBUILD_BUILD_DIR` keeps a stable build directory between runs, which
+improves incremental rebuilds and sccache hit rates.
+
+Check cache stats with:
+
+```bash
+sccache --show-stats
+```
+
 ## Testing
 
 We are using pytest for testing.  You can run the tests by executing

bench/ndarray/miniexpr-eval.py

@@ -0,0 +1,47 @@
+from time import time
+import blosc2
+import numpy as np
+import numexpr as ne
+
+N = 10_000
+# dtype= np.int32
+dtype= np.float32
+# dtype= np.float64
+cparams = blosc2.CParams(codec=blosc2.Codec.BLOSCLZ, clevel=1)
+
+t0 = time()
+# a = blosc2.ones((N, N), dtype=dtype, cparams=cparams)
+# a = blosc2.arange(np.prod((N, N)), shape=(N, N), dtype=dtype, cparams=cparams)
+a = blosc2.linspace(0., 1., np.prod((N, N)), shape=(N, N), dtype=dtype, cparams=cparams)
+print(f"Time to create data: {(time() - t0) * 1000 :.4f} ms")
+t0 = time()
+b = a.copy()
+c = a.copy()
+print(f"Time to copy data: {(time() - t0) * 1000 :.4f} ms")
+
+t0 = time()
+res = (2 * a**2 - 3 * b + c + 1.2).compute(cparams=cparams)
+t = time() - t0
+print(f"Time to evaluate: {t * 1000 :.4f} ms", end=" ")
+print(f"Speed (GB/s): {(a.nbytes * 4 / 1e9) / t:.2f}")
+# print(res.info)
+
+na = a[:]
+nb = b[:]
+nc = c[:]
+
+t0 = time()
+nres = 2 * na**2 - 3 * nb + nc + 1.2
+nt = time() - t0
+print(f"Time to evaluate with NumPy: {nt * 1000 :.4f} ms", end=" ")
+print(f"Speed (GB/s): {(na.nbytes * 4 / 1e9) / nt:.2f}")
+print(f"Speedup Blosc2 vs NumPy: {nt / t:.2f}x")
+np.testing.assert_allclose(res, nres, rtol=1e-5)
+
+t0 = time()
+neres = ne.evaluate("2 * na**2 - 3 * nb + nc + 1.2")
+net = time() - t0
+print(f"Time to evaluate with NumExpr: {net * 1000 :.4f} ms", end=" ")
+print(f"Speed (GB/s): {(na.nbytes * 4 / 1e9) / net:.2f}")
+print(f"Speedup Blosc2 vs NumExpr: {net / t:.2f}x")
+np.testing.assert_allclose(res, neres, rtol=1e-5)

bench/ndarray/miniexpr-reduct-sum-multi.py

@@ -0,0 +1,49 @@
+from time import time
+import blosc2
+import numpy as np
+import numexpr as ne
+
+N = 10_000
+dtype= np.float32
+cparams = blosc2.CParams(codec=blosc2.Codec.BLOSCLZ, clevel=1)
+
+t0 = time()
+#a = blosc2.ones((N, N), dtype=dtype, cparams=cparams)
+#a = blosc2.arange(np.prod((N, N)), shape=(N, N), dtype=dtype, cparams=cparams)
+a = blosc2.linspace(0., 1., np.prod((N, N)), shape=(N, N), dtype=dtype, cparams=cparams)
+#rng = np.random.default_rng(1234)
+#a = rng.integers(0, 2, size=(N, N), dtype=dtype)
+#a = blosc2.asarray(a, cparams=cparams, urlpath="a.b2nd", mode="w")
+print(f"Time to create data: {(time() - t0) * 1000 :.4f} ms")
+t0 = time()
+b = a.copy()
+c = a.copy()
+print(f"Time to copy data: {(time() - t0) * 1000 :.4f} ms")
+
+t0 = time()
+res = blosc2.sum(2 * a**2 - 3 * b + c + 1.2)
+t = time() - t0
+print(f"Time to evaluate: {t * 1000 :.4f} ms", end=" ")
+print(f"Speed (GB/s): {(a.nbytes * 3 / 1e9) / t:.2f}")
+print("Result:", res, "Mean:", res / (N * N))
+
+na = a[:]
+nb = b[:]
+nc = c[:]
+
+t0 = time()
+nres = np.sum(2 * na**2 - 3 * nb + nc + 1.2)
+nt = time() - t0
+print(f"Time to evaluate with NumPy: {nt * 1000 :.4f} ms", end=" ")
+print(f"Speed (GB/s): {(na.nbytes * 3 / 1e9) / nt:.2f}")
+print("Result:", nres, "Mean:", nres / (N * N))
+print(f"Speedup Blosc2 vs NumPy: {nt / t:.2f}x")
+assert np.allclose(res, nres)
+
+t0 = time()
+neres = ne.evaluate("sum(2 * na**2 - 3 * nb + nc + 1.2)")
+net = time() - t0
+print(f"Time to evaluate with NumExpr: {net * 1000 :.4f} ms", end=" ")
+print(f"Speed (GB/s): {(na.nbytes * 3 / 1e9) / net:.2f}")
+print("Result:", neres, "Mean:", neres / (N * N))
+print(f"Speedup Blosc2 vs NumExpr: {net / t:.2f}x")