From 3ac3ae6af48149358efa086172859955730e878b Mon Sep 17 00:00:00 2001 From: adarshh347 Date: Mon, 12 Oct 2026 22:13:21 +0530 Subject: [PATCH 1/2] bench(index): prototype two-pass IVF construction and memory study Add an isolated streaming prototype, real-module regression tests, and reviewed synthetic benchmark reports. Preserve production defaults and legacy dedup semantics; document ANN sampling and production-design limitations. Proposal: https://github.com/StarTrail-org/PixelRAG/issues/179 --- benchmarks/__init__.py | 1 + benchmarks/index_build_memory.md | 347 ++++++++++++++++++ benchmarks/index_build_memory.py | 234 ++++++++++++ ...ndex-build-memory-20261012-flat-32768.json | 142 +++++++ ...ndex-build-memory-20261012-flat-65536.json | 142 +++++++ .../index-build-memory-20261012-pq-32768.json | 142 +++++++ .../index-build-memory-20261012-pq-65536.json | 142 +++++++ benchmarks/streaming_ivf.py | 202 ++++++++++ tests/test_index_build_memory.py | 295 +++++++++++++++ 9 files changed, 1647 insertions(+) create mode 100644 benchmarks/__init__.py create mode 100644 benchmarks/index_build_memory.md create mode 100644 benchmarks/index_build_memory.py create mode 100644 benchmarks/results/index-build-memory-20261012-flat-32768.json create mode 100644 benchmarks/results/index-build-memory-20261012-flat-65536.json create mode 100644 benchmarks/results/index-build-memory-20261012-pq-32768.json create mode 100644 benchmarks/results/index-build-memory-20261012-pq-65536.json create mode 100644 benchmarks/streaming_ivf.py create mode 100644 tests/test_index_build_memory.py diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 0000000..177d930 --- /dev/null +++ b/benchmarks/__init__.py @@ -0,0 +1 @@ +"""Opt-in local experiments; not part of the production package.""" diff --git a/benchmarks/index_build_memory.md b/benchmarks/index_build_memory.md new file mode 100644 index 0000000..ea73332 --- /dev/null +++ b/benchmarks/index_build_memory.md @@ -0,0 +1,347 @@ +# Index build memory study: measured opt-in prototype + +## Status and scope + +Inspected on 2026-10-11 and rechecked on 2026-10-12 at `b4fe76301465eaee3d8ac75e0bec54664e53c072`. +GitHub's upstream `main` endpoint returned the same SHA. Reviewed open/closed +PR titles (up to 200 entries), plus PR #140's body and discussion. No newer +streaming builder was found. [PR #155](https://github.com/StarTrail-org/PixelRAG/pull/155) +is merged and already provides IVFPQ. [PR #140](https://github.com/StarTrail-org/PixelRAG/pull/140#issuecomment-5461339035) +is open; its maintainer discussion reports a structured-key dedup slowdown and +asks for a separate resolution. This prototype deliberately preserves the +current packed-key collisions and int64 overflow; it does not resolve that PR. + +**Bounded synthetic measurements (2026-10-12).** The focused legacy allocation +failure was reproduced, 47 prototype/neighboring tests passed, and four benchmark +configurations completed on the reviewed code using the existing lockfile. No embedding models were downloaded/trained, paid or embedding +APIs called, or production services touched. Only FAISS index training on local +generated data was performed. This remains a proposal, not production integration. + +Production files, CLI defaults, dependencies and lockfile are unchanged. +The experiment lives only in `benchmarks/`; tests import real NumPy, FAISS, +`pixelrag_embed.index`, and the prototype. Missing FAISS causes a collection +error in the new suite, not a silent skip. + +## Existing behavior + +`_merge_all_shards` loads the compressed `.npz` embeddings even during its shape +scan: `mmap_mode` does not memory-map compressed ZIP members. It then allocates +an `N_raw × d` float32 matrix, metadata arrays, packed keys and unique positions. +If there are duplicates, advanced indexing allocates another retained-vector +matrix before the raw matrix can be freed. `build_ivf` trains and adds in +100,000-row batches *after* this merge. PQ compresses the final index, not those +input matrices. `build_qdrant` also merges first and only then uploads batches. + +`FaissBackend` in `serve/src/pixelrag_serve/backends.py` uses FAISS row IDs to +index five `metadata.npz` arrays. Their names, dtypes and first-occurrence order +must remain aligned. This prototype preserves that format; it does not reduce +the serving process's metadata loading footprint. + +## Prototype design + +1. Sort `shard_*.npz` lexically. Read one shard at a time, casting metadata just + as the legacy merge does. Compute the same packed signed int64 key: + `article_id * 100_000_000 + tile_index * 10_000 + chunk_index`. +2. Insert each key into an on-disk SQLite primary-key table with a 2 MiB page + cache. Keep only the first occurrence. Save per-shard survivor positions to + disk. No corpus-sized Python set or in-memory metadata table is retained. +3. In that first pass, reservoir-sample surviving float32 vectors, with a fixed + seed and a configured maximum sample count. Train the same CPU IVFFlat or + IVFPQ classes and explicitly retain the requested IP/L2 metric. +4. Release the reservoir, reread shards, and add saved survivors in stable order + in bounded batches. Stream metadata into five raw files, then copy them in + bounded chunks into `.npy` members of a compatible uncompressed `.npz`. +5. Write index, metadata and summary in a temporary directory on the target + filesystem. Rename the completed directory into a **new** destination only. + Exceptions clean temporary output; an existing destination is rejected. + +This is a CPU-only experiment, not a reusable backend API. Per-row SQLite and +reservoir work can be slow. Their CPU and disk costs must be measured before +considering integration. Reservoir sampling is uniform without replacement, +but uses a different RNG/ordering from legacy `np.random.choice`; identical +seeds do not imply identical trained centroids or ANN neighbors. + +The bound is **not constant total memory**. Live memory includes one full +uncompressed shard (and conversion copies), a training sample or add batch, +SQLite's cache, per-shard selections/keys, FAISS training scratch and the final +in-memory index. The shard-path/signature list scales with the number of files. +The prototype does not stream inside an individual huge compressed archive. +A shard-size contract or a new chunkable input format would be needed for that. + +For IVFFlat, stored vectors alone cost about `4 * N_unique * d` bytes, plus +8-byte IDs, centroids, inverted-list capacity slack and other overhead. Eliminating +an extra raw input matrix does not eliminate this footprint. For PQ, codes cost +approximately `ceil(pq_m * pq_nbits / 8)` bytes/vector plus IDs and codebooks; +training/shard buffers may then dominate. These are storage estimates, not RSS +measurements or a prediction for a 30M-vector deployment. + +Temporary disk use includes the SQLite database/journal, 8-byte survivor row +positions, 24 bytes/vector of raw metadata, another metadata archive, and the +final index. Disk exhaustion must leave the destination unpublished. No resume +support is implemented. SIGKILL may leave a temporary directory; rename gives +visibility atomicity on the same filesystem, not fsync-backed crash durability. +Concurrent builders targeting the same path are unsupported. Inputs must be +immutable; checks after training and addition re-list the shards and compare +paths, file identities, sizes and mtimes. They detect common changes but are +not a locked snapshot or content-hash guarantee. + +## Reproduction + +Follow `CLAUDE.md` and CI's core-plus-dev setup. The dev extra supplies NumPy, +FAISS, pytest and local/mock backend testing dependencies without torch. The +locked sync below creates `.venv` without changing project dependencies or the +lockfile. Subsequent project commands intentionally use `--no-sync`; Ruff uses +CI's `uvx` workflow (0.17.0 for these checks). + +```sh +uv --version +uv sync --locked --extra dev --python 3.12 + +# Red: real legacy builder should hit the full-corpus allocation assertion. +PIXELRAG_STUDY_BASELINE=1 uv run --no-sync pytest -q \ + tests/test_index_build_memory.py::test_does_not_allocate_corpus_matrix + +# Green and neighboring builder/metadata-consumer coverage. +uv run --no-sync pytest -q tests/test_index_build_memory.py \ + tests/test_ivfpq_index.py tests/test_serve_backends.py \ + tests/test_serve_reconstruct.py tests/test_department_search.py + +# Each work directory must be absent. These only generate local random data. +uv run --no-sync python benchmarks/index_build_memory.py \ + --work-dir /tmp/pixelrag-memory-study-flat-review-20261012 +uv run --no-sync python benchmarks/index_build_memory.py \ + --work-dir /tmp/pixelrag-memory-study-pq-review-20261012 --pq-m 16 + +# Repeat at half the corpus volume with fixed shard/training sizes. +uv run --no-sync python benchmarks/index_build_memory.py \ + --work-dir /tmp/pixelrag-memory-study-flat-small-review-20261012 --rows 32768 +uv run --no-sync python benchmarks/index_build_memory.py \ + --work-dir /tmp/pixelrag-memory-study-pq-small-review-20261012 --rows 32768 --pq-m 16 + +uvx ruff check benchmarks/__init__.py benchmarks/streaming_ivf.py \ + benchmarks/index_build_memory.py tests/test_index_build_memory.py +uvx ruff format --check benchmarks/__init__.py \ + benchmarks/streaming_ivf.py benchmarks/index_build_memory.py \ + tests/test_index_build_memory.py +git diff --check +``` + +Default fixture: 65,536 rows × 256 dimensions, 4,096 rows/shard, normalized +float32 Gaussian vectors, about 5% duplicate keys with different vector/metadata +payloads, seed 2026. Raw vector volume is 64 MiB; generator memory is shard-sized. +The driver caps raw corpus, shard and reservoir matrices at 256 MiB each; this +is not a bound on whole-process RSS or total retained disk output. Default +settings use 8,192 training rows, 64 lists, nprobe 8, a 1,024-row prototype add +batch, one CPU thread and three repeats with alternating builder order. Legacy +keeps its production add batch; a local mock overrides only its CPU thread +count. PQ uses 16 subquantizers and 4 bits in the example above. + +Every builder invocation runs in a fresh interpreter. Generation and validation +are outside those processes. `build_wall_seconds` surrounds only the builder; +peak RSS is the process lifetime high-water mark including imports and build, +recorded before any validation. `resource.getrusage(...).ru_maxrss` is **bytes +on macOS, KiB on Linux**. The JSON preserves the raw value and unit, and converts +to MiB by dividing bytes by 1,048,576. Other platforms are rejected. Thread count, +Python/NumPy/FAISS versions and OS are recorded. Repeats share OS file caches; +these are not cold-cache measurements. No performance threshold is asserted. + +`results.json` contains per-run peak RSS, wall time, index/metadata/total output +bytes, retained vector count, all five metadata-array equality checks against +the legacy merge, ANN top-10 set overlap between builders, and overlap with an +exact `IndexFlatIP` oracle on 128 independently generated normalized queries. +For corpora below ten vectors, metric names report the actual `k = min(10, N)`. +It also reports overlap when all IVF lists are probed (exact for IVFFlat, still +quantized for PQ). Missing neighbor IDs never count as matches. Validation +intentionally allocates the complete *bounded fixture* outside measured workers. +The small pytest fixtures additionally reconstruct every IVFFlat row to verify +vector/metadata alignment, exercise duplicate keys/collisions/overflow, both +metrics and PQ, inject a failed index write and shard mutations, verify bounded +repeatable sampling/addition, and load output through the real serving backend +with normal and memory-mapped indexes. + +## Validation record (2026-10-12) + +| Check | Actual result | +| --- | --- | +| `uv --version` | 0.13.0 (Homebrew, aarch64-apple-darwin) | +| `uv sync --locked --extra dev --python 3.12` | Exit 0; CPython 3.12.13, 42 packages installed; lockfile unchanged | +| Baseline-mode focused test | Expected exit 1: **1 failed in 0.08s**, at production `index.py:78`, assertion `builder allocated the full raw input matrix`, shape `(384, 16)` | +| Prototype plus four neighboring suites (command above) | Exit 0: **47 passed in 0.73s, 0 skipped** | +| IVFFlat/PQ at 32,768/65,536 rows | All four commands exit 0; 24 fresh builder processes, 12 paired validations completed | +| Ruff lint and format checks | Exit 0; all lint checks passed, all four Python files formatted | +| `git diff --check` | Exit 0 | + +Review regressions were demonstrated before their fixes: adding a shard after +training did not raise (**1 failed in 0.22s**); a four-vector benchmark mislabeled +its overlap as top-10 (**1 failed in 0.09s**). Both pass in the final suite. +The shard checks now also reject same-size/mtime replacements and additions +during the second pass, leaving no published output. Tests verify a 96-row +training bound, 17-row add batches, first-wins sampling, deterministic seeds, +and both IVFFlat/PQ serving compatibility with normal and mmap loading. + +Diff review against upstream `b4fe763` found only this benchmark package, the +study/report artifacts and `tests/test_index_build_memory.py`; no production, +retrieval-expansion, rendering, dependency or lockfile changes are included. +No missing-import skips were taken. The full browser/ML suite and Linux runs +were not executed; the selected neighboring suites cover IVF/PQ, serving +backends, reconstruction and department search. + +## Measurements + +Hardware: Apple M4, 10 logical CPUs, 16 GiB RAM. OS: macOS 26.5.1 arm64. +Python 3.12.13, NumPy 2.4.5, FAISS 1.14.2. Each builder used one FAISS CPU +thread. Each reviewed configuration ran three pairs sequentially, alternating +builder order. The stored reports supersede the pre-review measurements. The +machine was not reserved exclusively for benchmarks; timings are short, share +filesystem caches, and should not be extrapolated to production. + +Values below are **median [minimum, maximum]**, across three repeats. Peak RSS +includes process imports and allocator retention, not just live vector buffers. + +| Index / raw rows | Legacy peak RSS, MiB | Prototype peak RSS, MiB | Legacy build, seconds | Prototype build, seconds | +| --- | --- | --- | --- | --- | +| IVFFlat / 32,768 | 171.72 [171.72, 171.75] | 107.83 [107.80, 107.84] | 0.217 [0.214, 0.218] | 0.197 [0.195, 0.197] | +| IVFFlat / 65,536 | 268.62 [268.61, 268.77] | 142.80 [142.70, 142.83] | 0.435 [0.428, 0.440] | 0.391 [0.388, 0.397] | +| IVFPQ / 32,768 | 166.11 [165.00, 166.12] | 73.20 [73.17, 73.59] | 0.305 [0.267, 0.317] | 0.259 [0.247, 0.278] | +| IVFPQ / 65,536 | 263.98 [263.97, 264.11] | 74.48 [74.44, 74.58] | 0.487 [0.485, 0.488] | 0.428 [0.426, 0.428] | + +At the larger volume, the observed median RSS difference is 125.83 MiB for +IVFFlat and 189.50 MiB for PQ. These are whole-builder comparisons, not a +component-level allocation profile. The legacy duplicate-removal copy matters: +this fixture has 5% duplicate keys, so both a 64 MiB raw matrix and a roughly +60.8 MiB retained matrix can coexist during dedup. A duplicate-free fixture +would avoid that particular copy and need its own measurements. + +With fixed shard/sample sizes, doubling rows increased prototype IVFFlat median +RSS by 34.97 MiB and its index file by about 30.64 MiB. PQ prototype median RSS +changed by 1.28 MiB. This is consistent with retaining the final IVFFlat index +while removing corpus-sized input buffers; it is not proof of a constant total +RSS bound or a prediction for 30M vectors. Larger and representative-data runs, +including dedup disk/CPU profiling, remain necessary before adoption. + +Both builders retained exactly **31,130 / 62,260 vectors** at the respective +volumes. All five metadata arrays, their order and dtypes matched the legacy +merge in every run. File sizes were repeat-stable and equal between builders +except for summary JSON (the prototype adds experiment/seed fields). + +| Index / raw rows | Index bytes, either builder | Metadata bytes, either builder | Total output bytes, legacy / prototype | +| --- | --- | --- | --- | +| IVFFlat / 32,768 | 32,192,347 | 748,416 | 32,941,133 / 32,941,204 | +| IVFFlat / 65,536 | 64,318,507 | 1,495,536 | 65,814,401 / 65,814,472 | +| IVFPQ / 32,768 | 580,692 | 748,416 | 1,329,470 / 1,329,541 | +| IVFPQ / 65,536 | 1,078,772 | 1,495,536 | 2,574,658 / 2,574,729 | + +Top-10 set overlap fractions below were identical across all three repeats. +Queries are independent normalized Gaussian vectors, not self-queries. + +| Index / raw rows | ANN vs exact, legacy / prototype | ANN between builders | All lists vs exact, legacy / prototype | +| --- | --- | --- | --- | +| IVFFlat / 32,768 | 0.274219 / 0.266406 | 0.227344 | 1.000000 / 1.000000 | +| IVFFlat / 65,536 | 0.278906 / 0.287500 | 0.228125 | 1.000000 / 1.000000 | +| IVFPQ / 32,768 | 0.008594 / 0.005469 | 0.000000 | 0.008594 / 0.005469 | +| IVFPQ / 65,536 | 0.010156 / 0.006250 | 0.001563 | 0.010938 / 0.006250 | + +All-list IVFFlat agreement and small-fixture reconstruction tests support row +alignment; approximate rankings are **not equivalent**. Different reservoir +samples/order change training. The deliberately small 16-subquantizer, 4-bit PQ +codes perform poorly on this high-dimensional random fixture; this setting is +not a production recommendation. Neither the slight IVFFlat differences nor PQ +results establish a semantic-retrieval improvement. Matched-sample training and +representative-data recall are separate required experiments. + +Raw reports (configuration, platform, each run and validation): + +- [IVFFlat, 32,768 rows](results/index-build-memory-20261012-flat-32768.json) +- [IVFFlat, 65,536 rows](results/index-build-memory-20261012-flat-65536.json) +- [IVFPQ, 32,768 rows](results/index-build-memory-20261012-pq-32768.json) +- [IVFPQ, 65,536 rows](results/index-build-memory-20261012-pq-65536.json) + +Generated shards, indexes and per-worker logs remain in the four `/tmp` work +directories listed in the reproduction commands; only small JSON reports are +included in the repository. Use new work-directory names to rerun. + +## Staged maintainer decision + +1. Review the completed red/green checks and four bounded studies, including + the poor synthetic PQ overlap. Reproduce on another machine and measure a + duplicate-free fixture and representative data before setting any resource + budget or performance target. Inspect dedup disk/CPU cost at larger volumes. +2. Decide the dedup identity/bounds separately with PR #140. This study preserves + current behavior, including known collisions. Compare disk-backed dedup cost + against alternatives before choosing a production implementation. Consider a + matched-training-sample experiment to isolate sampling-induced ANN variation. +3. Only after measurements, propose an explicit opt-in production builder with + supported resource budgets, input validation, snapshot/locking policy, disk + requirements and publication/recovery guarantees. Keep defaults until reviewed. +4. Consider external-memory FAISS inverted lists/sharded indexes separately if + the final IVFFlat index itself exceeds RAM. Consider Qdrant streaming in a + separate change: retain current UUID/payload mapping, first-wins input dedup, + append/recreate semantics, and address partial remote uploads/retries. No + Qdrant client is contacted in this study. + +Open questions: acceptable disk/CPU tradeoff; maximum shard size; minimum PQ +training volume; dedup migration; metadata format longevity; replacement and +crash-recovery policy; representative real-data recall evaluation. Synthetic +neighbor agreement is not evidence of semantic-retrieval quality. + +## Proposed PR (draft only, not submitted) + +**Title:** `bench(index): prototype two-pass IVF construction and memory study` + +**Body:** IVF/PQ builds currently merge the complete raw float32 corpus before +batched addition. Add an isolated two-pass prototype and bounded synthetic +benchmark using disk-backed legacy dedup, reservoir training, ordered addition +and compatible streamed metadata. Production defaults remain unchanged. + +Validation: reproduce the legacy allocation failure; 47 focused/neighboring tests +pass with no skips; Ruff and whitespace checks pass. Four configurations include +three fresh-process pairs each. At 65,536 × 256 rows, median RSS was +268.62 → 142.80 MiB (IVFFlat) and 263.98 → 74.48 MiB (PQ); raw reports include +wall time, counts, metadata, sizes and neighbor overlap. Limits: final index +memory remains, sampling changes ANN results, and random-data PQ overlap is poor. +The prototype requires immutable shards and a new output directory; Linux, +production-scale recall and crash recovery remain unvalidated. + +## Maintainer discussion draft (not posted) + +Would an opt-in bounded-input-memory builder be useful with this tradeoff? +The reviewed laptop runs remove much of the merge overhead while preserving +first-wins row order and the metadata format. IVFFlat still stores full vectors +in memory. SQLite moves dedup state to disk with a bounded page cache, but adds +per-row SQL work and scratch-space requirements that need measurement at scale. +I suggest keeping this as a benchmark/proposal until those costs are understood. + +Two policy decisions should stay separate. This study retains the packed-int64 +key semantics, including collisions/overflow, so it does not absorb #140. +Reservoir sampling is reproducible for a fixed seed but changes trained +centroids and ANN neighbors compared with the current sampling method. Is that +acceptable for an explicitly opt-in rebuild, or should a matched-training-sample +experiment be required first? Neither current overlap results nor the small +wall-time differences establish real-data retrieval or production speed gains. + +Before integration, I propose agreeing on maximum shard/training sizes, scratch +disk budget, and minimum representative-data recall evidence. The prototype +publishes only complete new directories; replacing an existing index, concurrent +writers, locked input snapshots and crash-durable recovery need their own policy. +If the final IVFFlat index is the RAM bottleneck, external-memory inverted lists +are a separate project. Qdrant streaming/retry semantics should also be reviewed +separately. No production default change is proposed by this contribution. + +## Proposed issue message (draft only, not posted) + +**Title:** Measure and design bounded-input-memory index construction + +`_merge_all_shards` allocates the complete raw float32 corpus before IVF/PQ or +Qdrant batching. PQ (#155) reduces final index storage but does not remove this +input allocation. Could we first review a bounded synthetic memory study and +an opt-in two-pass prototype before changing defaults? The draft compares fresh +processes on peak RSS, build time, output size, metadata order and neighbor +overlap, preserving existing packed-key dedup pending #140. On a 65,536 × 256 +synthetic fixture, three-repeat median RSS was 268.62 → 142.80 MiB for IVFFlat +and 263.98 → 74.48 MiB for PQ. Both retained 62,260 rows with identical metadata; +IVFFlat matched exact search when probing all lists, but ANN neighbors differ +and aggressive PQ has poor overlap on random data. These laptop measurements +are not production-scale or semantic-retrieval claims. Decisions needed include +dedup storage cost, shard/sample limits, publication and recovery, and whether +final IVFFlat memory requires a separate external-memory design. This proposal +does not include the #140 collision fix or a production backend rewrite. diff --git a/benchmarks/index_build_memory.py b/benchmarks/index_build_memory.py new file mode 100644 index 0000000..a323606 --- /dev/null +++ b/benchmarks/index_build_memory.py @@ -0,0 +1,234 @@ +"""Local generated-data benchmark. Run with uv; see index_build_memory.md.""" + +import argparse +import json +import os +import platform +import resource +import subprocess +import sys +import time +from pathlib import Path +from unittest.mock import patch + +import faiss +import numpy as np + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / "embed/src")) +from pixelrag_embed.index import _merge_all_shards, build_ivf + +from benchmarks.streaming_ivf import FIELDS, build_streaming_ivf + + +def generate_shards(destination, *, rows=65536, dim=256, shard_rows=4096, seed=2026): + """Normalized float32 noise, 5% repeated keys, distinct duplicate payloads.""" + destination.mkdir() + rng = np.random.default_rng(seed) + for shard, start in enumerate(range(0, rows, shard_rows)): + size = min(shard_rows, rows - start) + positions = np.arange(start, start + size, dtype=np.int64) + aids = positions.copy() + # Includes duplicates inside a shard and across shard boundaries. + duplicate = (positions > 0) & (positions % 20 == 0) + aids[duplicate] -= 1 + vectors = rng.standard_normal((size, dim)).astype(np.float32) + vectors /= np.linalg.norm(vectors, axis=1, keepdims=True) + np.savez_compressed( + destination / f"shard_{shard:06}.npz", + embeddings=vectors, + article_ids=aids, + tile_indices=np.zeros(size, dtype=np.int32), + chunk_indices=np.zeros(size, dtype=np.int32), + y_offsets=positions.astype(np.int32), + tile_heights=(100 + positions % 17).astype(np.int32), + ) + + +def worker(args): + faiss.omp_set_num_threads(args.threads) + np.random.seed(args.seed) + options = dict( + nlist=args.nlist, + nprobe=args.nprobe, + train_sample=args.train_sample, + pq_m=args.pq_m, + pq_nbits=4, + ) + started = time.perf_counter() + if args.worker == "legacy": + # Production resets FAISS threads from os.cpu_count(). Keep both runs + # on the same explicit CPU budget without editing production code. + with patch("pixelrag_embed.index.os.cpu_count", return_value=args.threads): + build_ivf(str(args.work_dir / "shards"), str(args.output), **options) + else: + build_streaming_ivf( + args.work_dir / "shards", + args.output, + seed=args.seed, + batch_size=args.batch_size, + **options, + ) + elapsed = time.perf_counter() - started + raw_rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + if sys.platform == "darwin": + unit, rss_bytes = "bytes", raw_rss + elif sys.platform.startswith("linux"): + unit, rss_bytes = "KiB", raw_rss * 1024 + else: + raise RuntimeError("Peak RSS conversion implemented only for macOS and Linux") + result = { + "builder": args.worker, + "build_wall_seconds": elapsed, + "peak_rss_raw": raw_rss, + "peak_rss_os_unit": unit, + "peak_rss_mib": rss_bytes / 1024**2, + "index_bytes": (args.output / "index.faiss").stat().st_size, + "metadata_bytes": (args.output / "metadata.npz").stat().st_size, + "total_output_bytes": sum(p.stat().st_size for p in args.output.iterdir()), + } + args.output.with_suffix(".metrics.json").write_text(json.dumps(result, indent=2)) + + +def overlap(left, right): + return float(np.mean([len(set(a) & set(b)) / len(a) for a, b in zip(left, right)])) + + +def validate(source, outputs, args): + # Deliberately outside timed/RSS-measured builder processes. This oracle + # allocates the complete fixture, which is why fixture size is capped. + merged = _merge_all_shards(sorted(source.glob("shard_*.npz"))) + vectors = merged["embeddings"] + rng = np.random.default_rng(args.seed + 1) + queries = rng.standard_normal((128, args.dim)).astype(np.float32) + queries /= np.linalg.norm(queries, axis=1, keepdims=True) + faiss.omp_set_num_threads(args.threads) + exact = faiss.IndexFlatIP(args.dim) + exact.add(vectors) + k = min(10, len(vectors)) + _, truth = exact.search(queries, k) + results, neighbors = {}, {} + for label, output in outputs.items(): + index = faiss.read_index(str(output / "index.faiss")) + assert index.ntotal == len(vectors) + with np.load(output / "metadata.npz") as metadata: + for name, dtype in FIELDS.items(): + np.testing.assert_array_equal(metadata[name], merged[name]) + assert metadata[name].dtype == dtype + _, ann = index.search(queries, k) + # Ignore missing neighbors when computing overlap with truth. + neighbors[label] = ann + index.nprobe = index.nlist + _, exhaustive = index.search(queries, k) + results[label] = { + "vector_count": index.ntotal, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + f"ann_overlap_at_{k}_with_exact": overlap(ann, truth), + f"all_lists_overlap_at_{k}_with_exact": overlap(exhaustive, truth), + } + del index + # Missing IDs must never count as common neighbors. + results[f"between_builders_ann_overlap_at_{k}"] = float( + np.mean( + [ + len((set(a) & set(b)) - {-1}) / k + for a, b in zip(neighbors["legacy"], neighbors["prototype"]) + ] + ) + ) + return results + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--work-dir", type=Path, required=True) + parser.add_argument("--rows", type=int, default=65536) + parser.add_argument("--dim", type=int, default=256) + parser.add_argument("--shard-rows", type=int, default=4096) + parser.add_argument("--train-sample", type=int, default=8192) + parser.add_argument("--batch-size", type=int, default=1024) + parser.add_argument("--nlist", type=int, default=64) + parser.add_argument("--nprobe", type=int, default=8) + parser.add_argument("--pq-m", type=int, default=0) + parser.add_argument("--threads", type=int, default=1) + parser.add_argument("--seed", type=int, default=2026) + parser.add_argument("--repeats", type=int, default=3) + parser.add_argument( + "--worker", choices=["legacy", "prototype"], help=argparse.SUPPRESS + ) + parser.add_argument("--output", type=Path, help=argparse.SUPPRESS) + args = parser.parse_args() + if args.worker: + worker(args) + return + for name in ( + "rows", + "dim", + "shard_rows", + "train_sample", + "batch_size", + "nlist", + "nprobe", + "threads", + "repeats", + ): + if getattr(args, name) < 1: + parser.error(f"{name} must be positive") + largest_matrix = max(args.rows, args.shard_rows, args.train_sample) * args.dim * 4 + if largest_matrix > 256 * 1024**2: + parser.error("Study capped at 256 MiB for raw corpus, shard and reservoir") + if args.pq_m < 0 or (args.pq_m and args.dim % args.pq_m): + parser.error("pq-m must be nonnegative and divide dim") + args.work_dir.mkdir(parents=True, exist_ok=False) + generate_shards( + args.work_dir / "shards", + rows=args.rows, + dim=args.dim, + shard_rows=args.shard_rows, + seed=args.seed, + ) + report = { + "configuration": { + k: str(v) if isinstance(v, Path) else v for k, v in vars(args).items() + }, + "platform": platform.platform(), + "python": sys.version, + "numpy": np.__version__, + "faiss": faiss.__version__, + "cpu_count": os.cpu_count(), + "runs": [], + } + for repeat in range(args.repeats): + outputs, metrics = {}, {} + # Alternate order to reduce systematic cache/order bias; no cold-cache claim. + order = ["legacy", "prototype"] if repeat % 2 == 0 else ["prototype", "legacy"] + for label in order: + output = args.work_dir / f"{label}-{repeat}" + outputs[label] = output + command = [ + sys.executable, + str(Path(__file__).resolve()), + *sys.argv[1:], + "--worker", + label, + "--output", + str(output), + ] + with (args.work_dir / f"{label}-{repeat}.log").open("w") as log: + subprocess.run( + command, check=True, stdout=log, stderr=subprocess.STDOUT + ) + metrics[label] = json.loads(output.with_suffix(".metrics.json").read_text()) + report["runs"].append( + { + "metrics": metrics, + "validation": validate(args.work_dir / "shards", outputs, args), + } + ) + (args.work_dir / "results.json").write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps(report, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/results/index-build-memory-20261012-flat-32768.json b/benchmarks/results/index-build-memory-20261012-flat-32768.json new file mode 100644 index 0000000..c2e51a1 --- /dev/null +++ b/benchmarks/results/index-build-memory-20261012-flat-32768.json @@ -0,0 +1,142 @@ +{ + "configuration": { + "work_dir": "/tmp/pixelrag-memory-study-flat-small-review-20261012", + "rows": 32768, + "dim": 256, + "shard_rows": 4096, + "train_sample": 8192, + "batch_size": 1024, + "nlist": 64, + "nprobe": 8, + "pq_m": 0, + "threads": 1, + "seed": 2026, + "repeats": 3, + "worker": null, + "output": null + }, + "platform": "macOS-26.5.1-arm64-arm-64bit", + "python": "3.12.13 (main, Mar 3 2026, 12:39:30) [Clang 21.0.0 (clang-2100.0.123.102)]", + "numpy": "2.4.5", + "faiss": "1.14.2", + "cpu_count": 10, + "runs": [ + { + "metrics": { + "legacy": { + "builder": "legacy", + "build_wall_seconds": 0.213675291999607, + "peak_rss_raw": 180060160, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 171.71875, + "index_bytes": 32192347, + "metadata_bytes": 748416, + "total_output_bytes": 32941133 + }, + "prototype": { + "builder": "prototype", + "build_wall_seconds": 0.19507695799984504, + "peak_rss_raw": 113033216, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 107.796875, + "index_bytes": 32192347, + "metadata_bytes": 748416, + "total_output_bytes": 32941204 + } + }, + "validation": { + "legacy": { + "vector_count": 31130, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.27421875, + "all_lists_overlap_at_10_with_exact": 1.0 + }, + "prototype": { + "vector_count": 31130, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.26640625, + "all_lists_overlap_at_10_with_exact": 1.0 + }, + "between_builders_ann_overlap_at_10": 0.22734375 + } + }, + { + "metrics": { + "prototype": { + "builder": "prototype", + "build_wall_seconds": 0.1968206250003277, + "peak_rss_raw": 113065984, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 107.828125, + "index_bytes": 32192347, + "metadata_bytes": 748416, + "total_output_bytes": 32941204 + }, + "legacy": { + "builder": "legacy", + "build_wall_seconds": 0.21834212500016292, + "peak_rss_raw": 180060160, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 171.71875, + "index_bytes": 32192347, + "metadata_bytes": 748416, + "total_output_bytes": 32941133 + } + }, + "validation": { + "prototype": { + "vector_count": 31130, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.26640625, + "all_lists_overlap_at_10_with_exact": 1.0 + }, + "legacy": { + "vector_count": 31130, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.27421875, + "all_lists_overlap_at_10_with_exact": 1.0 + }, + "between_builders_ann_overlap_at_10": 0.22734375 + } + }, + { + "metrics": { + "legacy": { + "builder": "legacy", + "build_wall_seconds": 0.21712562500033528, + "peak_rss_raw": 180092928, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 171.75, + "index_bytes": 32192347, + "metadata_bytes": 748416, + "total_output_bytes": 32941133 + }, + "prototype": { + "builder": "prototype", + "build_wall_seconds": 0.19673404100012704, + "peak_rss_raw": 113082368, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 107.84375, + "index_bytes": 32192347, + "metadata_bytes": 748416, + "total_output_bytes": 32941204 + } + }, + "validation": { + "legacy": { + "vector_count": 31130, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.27421875, + "all_lists_overlap_at_10_with_exact": 1.0 + }, + "prototype": { + "vector_count": 31130, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.26640625, + "all_lists_overlap_at_10_with_exact": 1.0 + }, + "between_builders_ann_overlap_at_10": 0.22734375 + } + } + ] +} diff --git a/benchmarks/results/index-build-memory-20261012-flat-65536.json b/benchmarks/results/index-build-memory-20261012-flat-65536.json new file mode 100644 index 0000000..8ed854a --- /dev/null +++ b/benchmarks/results/index-build-memory-20261012-flat-65536.json @@ -0,0 +1,142 @@ +{ + "configuration": { + "work_dir": "/tmp/pixelrag-memory-study-flat-review-20261012", + "rows": 65536, + "dim": 256, + "shard_rows": 4096, + "train_sample": 8192, + "batch_size": 1024, + "nlist": 64, + "nprobe": 8, + "pq_m": 0, + "threads": 1, + "seed": 2026, + "repeats": 3, + "worker": null, + "output": null + }, + "platform": "macOS-26.5.1-arm64-arm-64bit", + "python": "3.12.13 (main, Mar 3 2026, 12:39:30) [Clang 21.0.0 (clang-2100.0.123.102)]", + "numpy": "2.4.5", + "faiss": "1.14.2", + "cpu_count": 10, + "runs": [ + { + "metrics": { + "legacy": { + "builder": "legacy", + "build_wall_seconds": 0.4404755830000795, + "peak_rss_raw": 281657344, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 268.609375, + "index_bytes": 64318507, + "metadata_bytes": 1495536, + "total_output_bytes": 65814401 + }, + "prototype": { + "builder": "prototype", + "build_wall_seconds": 0.3880637920001391, + "peak_rss_raw": 149766144, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 142.828125, + "index_bytes": 64318507, + "metadata_bytes": 1495536, + "total_output_bytes": 65814472 + } + }, + "validation": { + "legacy": { + "vector_count": 62260, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.27890625, + "all_lists_overlap_at_10_with_exact": 1.0 + }, + "prototype": { + "vector_count": 62260, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.28750000000000003, + "all_lists_overlap_at_10_with_exact": 1.0 + }, + "between_builders_ann_overlap_at_10": 0.228125 + } + }, + { + "metrics": { + "prototype": { + "builder": "prototype", + "build_wall_seconds": 0.39737841700025456, + "peak_rss_raw": 149733376, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 142.796875, + "index_bytes": 64318507, + "metadata_bytes": 1495536, + "total_output_bytes": 65814472 + }, + "legacy": { + "builder": "legacy", + "build_wall_seconds": 0.43482858300012595, + "peak_rss_raw": 281673728, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 268.625, + "index_bytes": 64318507, + "metadata_bytes": 1495536, + "total_output_bytes": 65814401 + } + }, + "validation": { + "prototype": { + "vector_count": 62260, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.28750000000000003, + "all_lists_overlap_at_10_with_exact": 1.0 + }, + "legacy": { + "vector_count": 62260, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.27890625, + "all_lists_overlap_at_10_with_exact": 1.0 + }, + "between_builders_ann_overlap_at_10": 0.228125 + } + }, + { + "metrics": { + "legacy": { + "builder": "legacy", + "build_wall_seconds": 0.4282960419996016, + "peak_rss_raw": 281821184, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 268.765625, + "index_bytes": 64318507, + "metadata_bytes": 1495536, + "total_output_bytes": 65814401 + }, + "prototype": { + "builder": "prototype", + "build_wall_seconds": 0.39086562500051514, + "peak_rss_raw": 149635072, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 142.703125, + "index_bytes": 64318507, + "metadata_bytes": 1495536, + "total_output_bytes": 65814472 + } + }, + "validation": { + "legacy": { + "vector_count": 62260, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.27890625, + "all_lists_overlap_at_10_with_exact": 1.0 + }, + "prototype": { + "vector_count": 62260, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.28750000000000003, + "all_lists_overlap_at_10_with_exact": 1.0 + }, + "between_builders_ann_overlap_at_10": 0.228125 + } + } + ] +} diff --git a/benchmarks/results/index-build-memory-20261012-pq-32768.json b/benchmarks/results/index-build-memory-20261012-pq-32768.json new file mode 100644 index 0000000..4464753 --- /dev/null +++ b/benchmarks/results/index-build-memory-20261012-pq-32768.json @@ -0,0 +1,142 @@ +{ + "configuration": { + "work_dir": "/tmp/pixelrag-memory-study-pq-small-review-20261012", + "rows": 32768, + "dim": 256, + "shard_rows": 4096, + "train_sample": 8192, + "batch_size": 1024, + "nlist": 64, + "nprobe": 8, + "pq_m": 16, + "threads": 1, + "seed": 2026, + "repeats": 3, + "worker": null, + "output": null + }, + "platform": "macOS-26.5.1-arm64-arm-64bit", + "python": "3.12.13 (main, Mar 3 2026, 12:39:30) [Clang 21.0.0 (clang-2100.0.123.102)]", + "numpy": "2.4.5", + "faiss": "1.14.2", + "cpu_count": 10, + "runs": [ + { + "metrics": { + "legacy": { + "builder": "legacy", + "build_wall_seconds": 0.2674385410000468, + "peak_rss_raw": 173015040, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 165.0, + "index_bytes": 580692, + "metadata_bytes": 748416, + "total_output_bytes": 1329470 + }, + "prototype": { + "builder": "prototype", + "build_wall_seconds": 0.24723183300011442, + "peak_rss_raw": 76726272, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 73.171875, + "index_bytes": 580692, + "metadata_bytes": 748416, + "total_output_bytes": 1329541 + } + }, + "validation": { + "legacy": { + "vector_count": 31130, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.00859375, + "all_lists_overlap_at_10_with_exact": 0.00859375 + }, + "prototype": { + "vector_count": 31130, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.00546875, + "all_lists_overlap_at_10_with_exact": 0.00546875 + }, + "between_builders_ann_overlap_at_10": 0.0 + } + }, + { + "metrics": { + "prototype": { + "builder": "prototype", + "build_wall_seconds": 0.25919204099955095, + "peak_rss_raw": 76759040, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 73.203125, + "index_bytes": 580692, + "metadata_bytes": 748416, + "total_output_bytes": 1329541 + }, + "legacy": { + "builder": "legacy", + "build_wall_seconds": 0.30470995799987577, + "peak_rss_raw": 174194688, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 166.125, + "index_bytes": 580692, + "metadata_bytes": 748416, + "total_output_bytes": 1329470 + } + }, + "validation": { + "prototype": { + "vector_count": 31130, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.00546875, + "all_lists_overlap_at_10_with_exact": 0.00546875 + }, + "legacy": { + "vector_count": 31130, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.00859375, + "all_lists_overlap_at_10_with_exact": 0.00859375 + }, + "between_builders_ann_overlap_at_10": 0.0 + } + }, + { + "metrics": { + "legacy": { + "builder": "legacy", + "build_wall_seconds": 0.31715874999918015, + "peak_rss_raw": 174178304, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 166.109375, + "index_bytes": 580692, + "metadata_bytes": 748416, + "total_output_bytes": 1329470 + }, + "prototype": { + "builder": "prototype", + "build_wall_seconds": 0.2779058329997497, + "peak_rss_raw": 77168640, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 73.59375, + "index_bytes": 580692, + "metadata_bytes": 748416, + "total_output_bytes": 1329541 + } + }, + "validation": { + "legacy": { + "vector_count": 31130, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.00859375, + "all_lists_overlap_at_10_with_exact": 0.00859375 + }, + "prototype": { + "vector_count": 31130, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.00546875, + "all_lists_overlap_at_10_with_exact": 0.00546875 + }, + "between_builders_ann_overlap_at_10": 0.0 + } + } + ] +} diff --git a/benchmarks/results/index-build-memory-20261012-pq-65536.json b/benchmarks/results/index-build-memory-20261012-pq-65536.json new file mode 100644 index 0000000..9e91158 --- /dev/null +++ b/benchmarks/results/index-build-memory-20261012-pq-65536.json @@ -0,0 +1,142 @@ +{ + "configuration": { + "work_dir": "/tmp/pixelrag-memory-study-pq-review-20261012", + "rows": 65536, + "dim": 256, + "shard_rows": 4096, + "train_sample": 8192, + "batch_size": 1024, + "nlist": 64, + "nprobe": 8, + "pq_m": 16, + "threads": 1, + "seed": 2026, + "repeats": 3, + "worker": null, + "output": null + }, + "platform": "macOS-26.5.1-arm64-arm-64bit", + "python": "3.12.13 (main, Mar 3 2026, 12:39:30) [Clang 21.0.0 (clang-2100.0.123.102)]", + "numpy": "2.4.5", + "faiss": "1.14.2", + "cpu_count": 10, + "runs": [ + { + "metrics": { + "legacy": { + "builder": "legacy", + "build_wall_seconds": 0.48778383300032147, + "peak_rss_raw": 276938752, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 264.109375, + "index_bytes": 1078772, + "metadata_bytes": 1495536, + "total_output_bytes": 2574658 + }, + "prototype": { + "builder": "prototype", + "build_wall_seconds": 0.42783191699982126, + "peak_rss_raw": 78053376, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 74.4375, + "index_bytes": 1078772, + "metadata_bytes": 1495536, + "total_output_bytes": 2574729 + } + }, + "validation": { + "legacy": { + "vector_count": 62260, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.01015625, + "all_lists_overlap_at_10_with_exact": 0.0109375 + }, + "prototype": { + "vector_count": 62260, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.00625, + "all_lists_overlap_at_10_with_exact": 0.00625 + }, + "between_builders_ann_overlap_at_10": 0.0015625 + } + }, + { + "metrics": { + "prototype": { + "builder": "prototype", + "build_wall_seconds": 0.4263508340000044, + "peak_rss_raw": 78200832, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 74.578125, + "index_bytes": 1078772, + "metadata_bytes": 1495536, + "total_output_bytes": 2574729 + }, + "legacy": { + "builder": "legacy", + "build_wall_seconds": 0.4867744590001166, + "peak_rss_raw": 276807680, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 263.984375, + "index_bytes": 1078772, + "metadata_bytes": 1495536, + "total_output_bytes": 2574658 + } + }, + "validation": { + "prototype": { + "vector_count": 62260, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.00625, + "all_lists_overlap_at_10_with_exact": 0.00625 + }, + "legacy": { + "vector_count": 62260, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.01015625, + "all_lists_overlap_at_10_with_exact": 0.0109375 + }, + "between_builders_ann_overlap_at_10": 0.0015625 + } + }, + { + "metrics": { + "legacy": { + "builder": "legacy", + "build_wall_seconds": 0.4852889160001723, + "peak_rss_raw": 276791296, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 263.96875, + "index_bytes": 1078772, + "metadata_bytes": 1495536, + "total_output_bytes": 2574658 + }, + "prototype": { + "builder": "prototype", + "build_wall_seconds": 0.4278490420001617, + "peak_rss_raw": 78102528, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 74.484375, + "index_bytes": 1078772, + "metadata_bytes": 1495536, + "total_output_bytes": 2574729 + } + }, + "validation": { + "legacy": { + "vector_count": 62260, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.01015625, + "all_lists_overlap_at_10_with_exact": 0.0109375 + }, + "prototype": { + "vector_count": 62260, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.00625, + "all_lists_overlap_at_10_with_exact": 0.00625 + }, + "between_builders_ann_overlap_at_10": 0.0015625 + } + } + ] +} diff --git a/benchmarks/streaming_ivf.py b/benchmarks/streaming_ivf.py new file mode 100644 index 0000000..0f94fb1 --- /dev/null +++ b/benchmarks/streaming_ivf.py @@ -0,0 +1,202 @@ +"""Experimental CPU-only builder; deliberately not wired into pixelrag's CLI. + +Memory scales with the largest decompressed shard, training reservoir, add batch, +SQLite cache, and final FAISS index, rather than a second corpus-sized matrix. +Input files must be immutable for the duration of the two passes. +""" + +import json +import os +import shutil +import sqlite3 +import tempfile +import zipfile +from contextlib import ExitStack +from pathlib import Path + +import faiss +import numpy as np + +FIELDS = { + "article_ids": np.dtype("int64"), + "tile_indices": np.dtype("int32"), + "chunk_indices": np.dtype("int32"), + "y_offsets": np.dtype("int32"), + "tile_heights": np.dtype("int32"), +} + + +def _read_shard(path, dim): + # mmap_mode cannot mmap members of a compressed .npz archive. + with np.load(path, allow_pickle=False) as data: + vectors = np.asarray(data["embeddings"], dtype=np.float32) + if vectors.ndim != 2 or vectors.shape[1] == 0: + raise ValueError(f"Invalid embeddings shape: {path}") + if dim is not None and vectors.shape[1] != dim: + raise ValueError(f"Dimension mismatch: {path}") + metadata = {name: data[name].astype(dtype) for name, dtype in FIELDS.items()} + if any(values.shape != (len(vectors),) for values in metadata.values()): + raise ValueError(f"Metadata length mismatch: {path}") + return vectors, metadata + + +def _signature(files): + signature = [] + for path in files: + stat = path.stat() + signature.append( + (path, stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns) + ) + return signature + + +def _check_inputs(embeddings_dir, signature): + # Re-glob: re-statting only the original paths silently ignores new shards. + files = sorted(Path(embeddings_dir).glob("shard_*.npz")) + if _signature(files) != signature: + raise ValueError("Input shards changed during build") + + +def _pack_metadata(scratch, destination, count): + # Write .npy members directly into a ZIP_STORED .npz, without mapping/touching + # all metadata pages. The existing serve backend can read this unchanged. + with zipfile.ZipFile(destination, "w", allowZip64=True) as archive: + for name, dtype in FIELDS.items(): + with archive.open(f"{name}.npy", "w", force_zip64=True) as target: + np.lib.format.write_array_header_1_0( + target, + {"descr": dtype.str, "fortran_order": False, "shape": (count,)}, + ) + with (scratch / f"{name}.raw").open("rb") as source: + shutil.copyfileobj(source, target, length=1024 * 1024) + + +def build_streaming_ivf( + embeddings_dir, + output_dir, + *, + nlist=64, + nprobe=8, + train_sample=8192, + batch_size=1024, + seed=0, + metric="ip", + pq_m=0, + pq_nbits=8, +): + """Two passes: first-wins dedup + reservoir training, then streaming add. + + Retains legacy *packed int64* collisions and overflow, intentionally. This + is not the collision fix discussed in PR #140. SQLite stores signed keys; + survivor positions are disk files, not a corpus-sized Python set or list. + Destination must not exist. Publication is a same-filesystem directory + rename; concurrent builders targeting the same destination are unsupported. + """ + if min(nlist, nprobe, train_sample, batch_size) < 1: + raise ValueError("nlist, nprobe, train_sample and batch_size must be positive") + if metric not in {"ip", "l2"} or pq_m < 0 or pq_nbits not in {4, 8}: + raise ValueError("Invalid metric or PQ configuration") + files = sorted(Path(embeddings_dir).glob("shard_*.npz")) + if not files: + raise ValueError("No shard files found") + signature = _signature(files) + output = Path(output_dir).absolute() + if output.exists(): + raise FileExistsError(output) + output.parent.mkdir(parents=True, exist_ok=True) + rng = np.random.default_rng(seed) + with tempfile.TemporaryDirectory(prefix=".ivf-study-", dir=output.parent) as temp: + scratch = Path(temp) + stage = scratch / "output" + stage.mkdir() + count, dim, reservoir = 0, None, None + db = sqlite3.connect(scratch / "seen.sqlite") + try: + db.execute("PRAGMA cache_size=-2048") # 2 MiB page cache + db.execute("PRAGMA temp_store=FILE") + db.execute("CREATE TABLE seen (key INTEGER PRIMARY KEY)") + for shard_id, path in enumerate(files): + vectors, metadata = _read_shard(path, dim) + if dim is None: + dim = vectors.shape[1] + if pq_m and dim % pq_m: + raise ValueError("pq_m must divide dimension evenly") + reservoir = np.empty((train_sample, dim), dtype=np.float32) + # Match legacy casts before arithmetic, including int64 wrap. + keys = ( + metadata["article_ids"] * 100_000_000 + + metadata["tile_indices"].astype(np.int64) * 10_000 + + metadata["chunk_indices"].astype(np.int64) + ) + selected = [] # bounded by one shard, not the corpus + for row, key in enumerate(keys): + cursor = db.execute( + "INSERT OR IGNORE INTO seen VALUES (?)", (int(key),) + ) + if not cursor.rowcount: + continue + selected.append(row) + count += 1 + slot = count - 1 if count <= train_sample else rng.integers(count) + if slot < train_sample: + reservoir[slot] = vectors[row] + db.commit() + np.save( + scratch / f"rows_{shard_id}.npy", + np.asarray(selected, dtype=np.int64), + ) + del vectors, metadata, keys, selected + finally: + db.close() + if count == 0: + raise ValueError("No vectors after deduplication") + metric_type = faiss.METRIC_INNER_PRODUCT if metric == "ip" else faiss.METRIC_L2 + quantizer = faiss.IndexFlatIP(dim) if metric == "ip" else faiss.IndexFlatL2(dim) + nlist = min(nlist, count) + if pq_m: + index = faiss.IndexIVFPQ(quantizer, dim, nlist, pq_m, pq_nbits, metric_type) + else: + index = faiss.IndexIVFFlat(quantizer, dim, nlist, metric_type) + index.train(reservoir[: min(train_sample, count)]) + del reservoir + _check_inputs(embeddings_dir, signature) + # Open only five metadata streams; their size is O(N) on disk. + with ExitStack() as stack: + streams = { + name: stack.enter_context((scratch / f"{name}.raw").open("wb")) + for name in FIELDS + } + for shard_id, path in enumerate(files): + vectors, metadata = _read_shard(path, dim) + rows = np.load(scratch / f"rows_{shard_id}.npy", allow_pickle=False) + for start in range(0, len(rows), batch_size): + selection = rows[start : start + batch_size] + index.add(np.ascontiguousarray(vectors[selection])) + for name, stream in streams.items(): + stream.write(metadata[name][selection].tobytes()) + del vectors, metadata, rows + _check_inputs(embeddings_dir, signature) + if index.ntotal != count: + raise RuntimeError("Index/metadata row count mismatch") + index.nprobe = nprobe + _pack_metadata(scratch, stage / "metadata.npz", count) + faiss.write_index(index, str(stage / "index.faiss")) + summary = { + "backend": "faiss", + "total_vectors": count, + "dimension": dim, + "nlist": nlist, + "nprobe": nprobe, + "metric": metric, + "index_type": "ivfpq" if pq_m else "ivfflat", + "pq_m": pq_m, + "pq_nbits": pq_nbits if pq_m else None, + "index_file": str(output / "index.faiss"), + "metadata_file": str(output / "metadata.npz"), + "experimental_builder": "two-pass-reservoir", + "seed": seed, + } + (stage / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") + if output.exists(): + raise FileExistsError(output) + os.rename(stage, output) diff --git a/tests/test_index_build_memory.py b/tests/test_index_build_memory.py new file mode 100644 index 0000000..fd01c6e --- /dev/null +++ b/tests/test_index_build_memory.py @@ -0,0 +1,295 @@ +"""Real-module checks for the opt-in study; missing FAISS is a collection error.""" + +import json +import os +import sys +from pathlib import Path +from types import SimpleNamespace + +import faiss +import numpy as np +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / "embed/src")) +from pixelrag_embed.index import _merge_all_shards, build_ivf +from pixelrag_serve.backends import FaissBackend + +from benchmarks.index_build_memory import generate_shards, validate +from benchmarks.streaming_ivf import FIELDS, build_streaming_ivf + + +@pytest.fixture(autouse=True) +def single_thread(): + previous = faiss.omp_get_max_threads() + faiss.omp_set_num_threads(1) + yield + faiss.omp_set_num_threads(previous) + + +@pytest.fixture +def shards(tmp_path): + source = tmp_path / "shards" + generate_shards(source, rows=384, dim=16, shard_rows=128, seed=7) + return source + + +def test_does_not_allocate_corpus_matrix(shards, tmp_path, monkeypatch): + original = np.empty + + def guarded_empty(shape, *args, **kwargs): + assert shape != (384, 16), "builder allocated the full raw input matrix" + return original(shape, *args, **kwargs) + + monkeypatch.setattr(np, "empty", guarded_empty) + # Reproducible red/green demonstration against the *real* legacy builder. + builder = ( + build_ivf + if os.environ.get("PIXELRAG_STUDY_BASELINE") == "1" + else build_streaming_ivf + ) + builder(str(shards), str(tmp_path / "out"), nlist=2, train_sample=96) + index = faiss.read_index(str(tmp_path / "out/index.faiss")) + assert index.ntotal == 365 + + +@pytest.mark.parametrize("pq_m", [0, 4]) +@pytest.mark.parametrize("metric", ["ip", "l2"]) +def test_metadata_order_and_neighbor_alignment(shards, tmp_path, pq_m, metric): + expected = _merge_all_shards(sorted(shards.glob("*.npz"))) + output = tmp_path / "out" + build_streaming_ivf( + shards, + output, + nlist=2, + nprobe=2, + train_sample=365, + batch_size=17, + pq_m=pq_m, + pq_nbits=4, + metric=metric, + ) + index = faiss.read_index(str(output / "index.faiss")) + assert index.ntotal == len(expected["embeddings"]) + assert index.metric_type == ( + faiss.METRIC_INNER_PRODUCT if metric == "ip" else faiss.METRIC_L2 + ) + with np.load(output / "metadata.npz") as actual: + for name, dtype in FIELDS.items(): + np.testing.assert_array_equal(actual[name], expected[name]) + assert actual[name].dtype == dtype + queries = expected["embeddings"][:16] + _, ids = index.search(queries, 5) + assert (ids >= 0).all() + if not pq_m: + # Verify vector/metadata alignment, not just the metadata file by itself. + index.make_direct_map() + for row in range(index.ntotal): + np.testing.assert_array_equal( + index.reconstruct(row), expected["embeddings"][row] + ) + exact = faiss.IndexFlatIP(16) if metric == "ip" else faiss.IndexFlatL2(16) + exact.add(expected["embeddings"]) + np.testing.assert_array_equal(ids, exact.search(queries, 5)[1]) + + +def test_legacy_collisions_overflow_and_first_wins_are_preserved(tmp_path): + source = tmp_path / "shards" + source.mkdir() + # (1,10000,0) collides with (2,0,0); (3,0,10000) with (3,1,0). + # Large article IDs exercise the current int64 arithmetic overflow too. + for shard in range(2): + np.savez_compressed( + source / f"shard_{shard}.npz", + embeddings=np.arange(24, dtype=np.float32).reshape(6, 4) + shard, + article_ids=np.array([1, 2, 3, 3, 2**62, 2**62], dtype=np.int64), + tile_indices=np.array([10000, 0, 0, 1, 0, 0], dtype=np.int32), + chunk_indices=np.array([0, 0, 10000, 0, 0, 0], dtype=np.int32), + y_offsets=np.arange(6, dtype=np.int32) + shard * 100, + tile_heights=np.full(6, 100 + shard, dtype=np.int32), + ) + expected = _merge_all_shards(sorted(source.glob("*.npz"))) + build_streaming_ivf(source, tmp_path / "out", nlist=1, train_sample=3) + index = faiss.read_index(str(tmp_path / "out/index.faiss")) + index.make_direct_map() + with np.load(tmp_path / "out/metadata.npz") as actual: + for name in FIELDS: + np.testing.assert_array_equal(actual[name], expected[name]) + for row in range(index.ntotal): + np.testing.assert_array_equal( + index.reconstruct(row), expected["embeddings"][row] + ) + + +def test_write_failure_does_not_publish_partial_output(shards, tmp_path, monkeypatch): + def fail_write(index, path): + Path(path).write_bytes(b"partial") + raise OSError("injected disk failure") + + monkeypatch.setattr(faiss, "write_index", fail_write) + with pytest.raises(OSError, match="injected disk failure"): + build_streaming_ivf(shards, tmp_path / "out", nlist=2, train_sample=96) + assert not (tmp_path / "out").exists() + assert not list(tmp_path.glob(".ivf-study-*")) + + +def test_existing_output_is_untouched(shards, tmp_path): + output = tmp_path / "out" + output.mkdir() + (output / "sentinel").write_text("keep") + with pytest.raises(FileExistsError): + build_streaming_ivf(shards, output) + assert (output / "sentinel").read_text() == "keep" + + +def test_invalid_dimension_does_not_publish(shards, tmp_path): + with np.load(shards / "shard_000001.npz") as data: + values = dict(data) + values["embeddings"] = values["embeddings"][:, :8] + np.savez_compressed(shards / "shard_000001.npz", **values) + with pytest.raises(ValueError, match="Dimension mismatch"): + build_streaming_ivf(shards, tmp_path / "out", train_sample=96) + assert not (tmp_path / "out").exists() + + +def test_empty_shards_fail_cleanly(tmp_path): + source = tmp_path / "empty" + source.mkdir() + np.savez( + source / "shard_000.npz", + embeddings=np.empty((0, 16), dtype=np.float32), + **{name: np.empty(0, dtype=dtype) for name, dtype in FIELDS.items()}, + ) + with pytest.raises(ValueError, match="No vectors"): + build_streaming_ivf(source, tmp_path / "out", train_sample=16) + assert not (tmp_path / "out").exists() + + +@pytest.mark.parametrize("operation", ["train", "add"]) +@pytest.mark.parametrize("mutation", ["new_shard", "replace_shard"]) +def test_changed_shards_do_not_publish( + shards, tmp_path, monkeypatch, operation, mutation +): + original = getattr(faiss.IndexIVFFlat, operation) + changed = False + + def run_then_change_shards(index, data): + nonlocal changed + original(index, data) + if changed: + return + changed = True + source = shards / "shard_000000.npz" + if mutation == "new_shard": + with np.load(source) as archive: + values = dict(archive) + values["article_ids"] += 1000 + np.savez_compressed(shards / "shard_000003.npz", **values) + else: + # Replacement with the same size/mtime still changes file identity. + stat = source.stat() + replacement = tmp_path / "replacement.npz" + replacement.write_bytes(source.read_bytes()) + os.utime(replacement, ns=(stat.st_atime_ns, stat.st_mtime_ns)) + os.replace(replacement, source) + + monkeypatch.setattr(faiss.IndexIVFFlat, operation, run_then_change_shards) + with pytest.raises(ValueError, match="Input shards changed"): + build_streaming_ivf(shards, tmp_path / "out", nlist=2, train_sample=96) + assert not (tmp_path / "out").exists() + assert not list(tmp_path.glob(".ivf-study-*")) + + +def test_sampling_and_addition_are_bounded_and_repeatable( + shards, tmp_path, monkeypatch +): + expected = _merge_all_shards(sorted(shards.glob("*.npz")))["embeddings"] + train = faiss.IndexIVFFlat.train + add = faiss.IndexIVFFlat.add + samples, additions = [], [] + + def capture_train(index, data): + samples.append(data.copy()) + train(index, data) + + def capture_add(index, data): + assert 0 < len(data) <= 17 + additions.append(data.copy()) + add(index, data) + + monkeypatch.setattr(faiss.IndexIVFFlat, "train", capture_train) + monkeypatch.setattr(faiss.IndexIVFFlat, "add", capture_add) + for run in range(2): + additions.clear() + build_streaming_ivf( + shards, + tmp_path / f"out-{run}", + nlist=2, + train_sample=96, + batch_size=17, + seed=7, + ) + np.testing.assert_array_equal(np.concatenate(additions), expected) + np.testing.assert_array_equal(samples[0], samples[1]) + assert samples[0].shape == (96, 16) + survivors = {row.tobytes() for row in expected} + sampled = {row.tobytes() for row in samples[0]} + assert len(sampled) == 96 + assert sampled <= survivors + # Training must draw beyond the first shard, not just take an early prefix. + assert sampled & {row.tobytes() for row in expected[-100:]} + + +@pytest.mark.parametrize("pq_m", [0, 4]) +@pytest.mark.parametrize("mmap", [False, True]) +def test_output_is_compatible_with_serving_backend( + shards, tmp_path, monkeypatch, pq_m, mmap +): + monkeypatch.delenv("PIXELRAG_INDEX_MMAP", raising=False) + if mmap: + monkeypatch.setenv("PIXELRAG_INDEX_MMAP", "1") + expected = _merge_all_shards(sorted(shards.glob("*.npz"))) + output = tmp_path / "out" + build_streaming_ivf( + shards, + output, + nlist=2, + nprobe=2, + train_sample=365, + pq_m=pq_m, + pq_nbits=4, + ) + summary = json.loads((output / "summary.json").read_text()) + backend = FaissBackend(str(output), summary) + try: + assert backend.ntotal == len(expected["embeddings"]) + assert backend.dimension == 16 + assert backend.nprobe == 2 + query = expected["embeddings"][:1] + article = int(expected["article_ids"][0]) + hits = backend.raw_search(query, 5, article_ids=[article])[0] + assert len(hits) == 1 + hit = hits[0] + assert hit["vector_id"] == 0 + for stored, served in zip( + FIELDS, + ("article_id", "tile_index", "chunk_index", "y_offset", "tile_height"), + ): + assert hit[served] == int(expected[stored][0]) + finally: + backend._meta.close() + + +def test_small_benchmark_reports_actual_neighbor_count(tmp_path, monkeypatch): + source = tmp_path / "shards" + generate_shards(source, rows=4, dim=8, shard_rows=2, seed=7) + monkeypatch.setattr("pixelrag_embed.index.os.cpu_count", lambda: 1) + outputs = {name: tmp_path / name for name in ("legacy", "prototype")} + for name, builder in (("legacy", build_ivf), ("prototype", build_streaming_ivf)): + builder(source, outputs[name], nlist=1, nprobe=1, train_sample=4) + result = validate(source, outputs, SimpleNamespace(dim=8, seed=7, threads=1)) + assert result["between_builders_ann_overlap_at_4"] == 1.0 + for name in outputs: + assert result[name]["ann_overlap_at_4_with_exact"] == 1.0 + assert "ann_overlap_at_10_with_exact" not in result[name] From 77197653d4c46d94c78a65451f9361350516f041 Mon Sep 17 00:00:00 2001 From: adarshh347 Date: Wed, 18 Nov 2026 11:17:57 +0530 Subject: [PATCH 2/2] bench(index): refresh streaming memory study on current dedup --- benchmarks/index_build_memory.md | 164 ++++++++++++------ ...ndex-build-memory-20261118-flat-32768.json | 142 +++++++++++++++ ...ndex-build-memory-20261118-flat-65536.json | 142 +++++++++++++++ .../index-build-memory-20261118-pq-32768.json | 142 +++++++++++++++ .../index-build-memory-20261118-pq-65536.json | 142 +++++++++++++++ benchmarks/streaming_ivf.py | 55 ++++-- tests/test_index_build_memory.py | 49 +++++- 7 files changed, 756 insertions(+), 80 deletions(-) create mode 100644 benchmarks/results/index-build-memory-20261118-flat-32768.json create mode 100644 benchmarks/results/index-build-memory-20261118-flat-65536.json create mode 100644 benchmarks/results/index-build-memory-20261118-pq-32768.json create mode 100644 benchmarks/results/index-build-memory-20261118-pq-65536.json diff --git a/benchmarks/index_build_memory.md b/benchmarks/index_build_memory.md index ea73332..cda672c 100644 --- a/benchmarks/index_build_memory.md +++ b/benchmarks/index_build_memory.md @@ -2,20 +2,25 @@ ## Status and scope -Inspected on 2026-10-11 and rechecked on 2026-10-12 at `b4fe76301465eaee3d8ac75e0bec54664e53c072`. -GitHub's upstream `main` endpoint returned the same SHA. Reviewed open/closed -PR titles (up to 200 entries), plus PR #140's body and discussion. No newer -streaming builder was found. [PR #155](https://github.com/StarTrail-org/PixelRAG/pull/155) -is merged and already provides IVFPQ. [PR #140](https://github.com/StarTrail-org/PixelRAG/pull/140#issuecomment-5461339035) -is open; its maintainer discussion reports a structured-key dedup slowdown and -asks for a separate resolution. This prototype deliberately preserves the -current packed-key collisions and int64 overflow; it does not resolve that PR. - -**Bounded synthetic measurements (2026-10-12).** The focused legacy allocation -failure was reproduced, 47 prototype/neighboring tests passed, and four benchmark -configurations completed on the reviewed code using the existing lockfile. No embedding models were downloaded/trained, paid or embedding -APIs called, or production services touched. Only FAISS index training on local -generated data was performed. This remains a proposal, not production integration. +Originally measured on 2026-10-12 at `b4fe76301465eaee3d8ac75e0bec54664e53c072`. +Rebased and rechecked on 2026-11-18 against current upstream `main` at +`94fdad277ed0e64ecc217b935204e704a1d582a5` (40 commits ahead of that baseline). +Upstream merged [PR #203](https://github.com/StarTrail-org/PixelRAG/pull/203), +which replaced the old fixed-decimal dedup key with range-derived unique keys +and rejects ranges that cannot fit in `int64`. This study now implements the +same tuple identity using SQLite's composite primary key, checks the same range +limit, and preserves first-wins order. The 2026-10-12 reports are kept as +historical results; current-head measurements below supersede them. + +**Rebased synthetic measurements (2026-11-18).** The current production builder +still allocates the full raw vector matrix before building an IVF index. The +focused old-prototype failure was reproduced after the upstream rebase, the +prototype was corrected, 21 focused tests passed, and four benchmark +configurations completed on the current code using the existing lockfile. No +embedding models were downloaded/trained, paid or embedding APIs called, or +production services touched. Only FAISS index training on locally generated +data was performed. This remains an experimental study, not production +integration. Production files, CLI defaults, dependencies and lockfile are unchanged. The experiment lives only in `benchmarks/`; tests import real NumPy, FAISS, @@ -26,7 +31,8 @@ error in the new suite, not a silent skip. `_merge_all_shards` loads the compressed `.npz` embeddings even during its shape scan: `mmap_mode` does not memory-map compressed ZIP members. It then allocates -an `N_raw × d` float32 matrix, metadata arrays, packed keys and unique positions. +an `N_raw × d` float32 matrix, metadata arrays, range-derived packed keys and +unique positions. If there are duplicates, advanced indexing allocates another retained-vector matrix before the raw matrix can be freed. `build_ivf` trains and adds in 100,000-row batches *after* this merge. PQ compresses the final index, not those @@ -40,11 +46,14 @@ the serving process's metadata loading footprint. ## Prototype design 1. Sort `shard_*.npz` lexically. Read one shard at a time, casting metadata just - as the legacy merge does. Compute the same packed signed int64 key: - `article_id * 100_000_000 + tile_index * 10_000 + chunk_index`. -2. Insert each key into an on-disk SQLite primary-key table with a 2 MiB page - cache. Keep only the first occurrence. Save per-shard survivor positions to - disk. No corpus-sized Python set or in-memory metadata table is retained. + as the legacy merge does. Track global min/max for `article_id`, `tile_index`, + and `chunk_index` so the prototype can apply the same `int64` range limit as + production's `_dedup_keys`. +2. Insert each metadata tuple into an on-disk SQLite composite-primary-key table + with a 2 MiB page cache. Distinct tuples remain distinct even if they collided + under the retired fixed-decimal formula. Keep only the first occurrence and + save per-shard survivor positions to disk. No corpus-sized Python set or + in-memory metadata table is retained. 3. In that first pass, reservoir-sample surviving float32 vectors, with a fixed seed and a configured maximum sample count. Train the same CPU IVFFlat or IVFPQ classes and explicitly retain the requested IP/L2 metric. @@ -55,9 +64,9 @@ the serving process's metadata loading footprint. filesystem. Rename the completed directory into a **new** destination only. Exceptions clean temporary output; an existing destination is rejected. -This is a CPU-only experiment, not a reusable backend API. Per-row SQLite and -reservoir work can be slow. Their CPU and disk costs must be measured before -considering integration. Reservoir sampling is uniform without replacement, +This is a CPU-only experiment, not a reusable backend API. Per-row SQLite tuple +inserts and reservoir work can be slow. Their CPU and disk costs must be measured +before considering integration. Reservoir sampling is uniform without replacement, but uses a different RNG/ordering from legacy `np.random.choice`; identical seeds do not imply identical trained centroids or ANN neighbors. @@ -154,12 +163,12 @@ It also reports overlap when all IVF lists are probed (exact for IVFFlat, still quantized for PQ). Missing neighbor IDs never count as matches. Validation intentionally allocates the complete *bounded fixture* outside measured workers. The small pytest fixtures additionally reconstruct every IVFFlat row to verify -vector/metadata alignment, exercise duplicate keys/collisions/overflow, both -metrics and PQ, inject a failed index write and shard mutations, verify bounded +vector/metadata alignment, exercise duplicate keys, former fixed-key collisions, +the current range limit, both metrics and PQ, inject a failed index write and shard mutations, verify bounded repeatable sampling/addition, and load output through the real serving backend with normal and memory-mapped indexes. -## Validation record (2026-10-12) +## Historical validation record (2026-10-12) | Check | Actual result | | --- | --- | @@ -186,7 +195,7 @@ No missing-import skips were taken. The full browser/ML suite and Linux runs were not executed; the selected neighboring suites cover IVF/PQ, serving backends, reconstruction and department search. -## Measurements +## Historical measurements (2026-10-12) Hardware: Apple M4, 10 logical CPUs, 16 GiB RAM. OS: macOS 26.5.1 arm64. Python 3.12.13, NumPy 2.4.5, FAISS 1.14.2. Each builder used one FAISS CPU @@ -260,16 +269,55 @@ Generated shards, indexes and per-worker logs remain in the four `/tmp` work directories listed in the reproduction commands; only small JSON reports are included in the repository. Use new work-directory names to rerun. +## Current-head validation and measurements (2026-11-18) + +Current upstream: `94fdad277ed0e64ecc217b935204e704a1d582a5`. Rebased the +isolated study commit onto that SHA. The original focused suite had **19 passed +and 1 failed**: its old fixture combined extreme article IDs with broad tile +and chunk ranges, so production correctly raised the new #203 range-limit +error. The prototype then used the pre-#203 fixed-decimal key. Replaced it with +an on-disk SQLite composite key, kept the same production `int64` range check, +and rewrote the fixture to verify three former collisions remain distinct and +exact duplicates remain first-wins. Final focused suite: **21 passed**. + +The current-head benchmark used the same bounded generated fixtures and three +fresh-process builder pairs per configuration, alternating builder order. The +SQLite composite-key cost is included in the prototype times. Values are median +**[minimum, maximum]** over three runs: + +| Index / raw rows | Legacy peak RSS, MiB | Prototype peak RSS, MiB | Legacy build, seconds | Prototype build, seconds | +| --- | --- | --- | --- | --- | +| IVFFlat / 32,768 | 172.72 [172.55, 173.39] | 109.58 [108.94, 111.77] | 0.253 [0.244, 0.269] | 0.245 [0.228, 0.270] | +| IVFFlat / 65,536 | 272.73 [271.66, 276.30] | 144.52 [143.45, 148.66] | 0.561 [0.504, 0.566] | 0.501 [0.484, 0.528] | +| IVFPQ / 32,768 | 165.89 [165.86, 166.03] | 75.44 [74.55, 75.48] | 0.288 [0.286, 0.299] | 0.286 [0.268, 0.294] | +| IVFPQ / 65,536 | 264.75 [264.28, 265.20] | 76.23 [75.69, 77.14] | 0.529 [0.520, 0.532] | 0.465 [0.465, 0.491] | + +All five metadata arrays, dtypes, and row order matched the current production +merge in all four configurations. Retained rows were 31,130 and 62,260. +IVFFlat's all-list search matched exact top-10 on these generated queries; +approximate ANN rankings differed. PQ's random-data overlap remained poor. The +larger corpus reduced median peak RSS by 128.22 MiB for IVFFlat and 188.52 MiB +for PQ. These small shared-machine measurements do not establish production +scale or semantic-retrieval benefits. + +Current-head raw reports (include run configuration, platform and per-run +metrics): + +- [IVFFlat, 32,768 rows](results/index-build-memory-20261118-flat-32768.json) +- [IVFFlat, 65,536 rows](results/index-build-memory-20261118-flat-65536.json) +- [IVFPQ, 32,768 rows](results/index-build-memory-20261118-pq-32768.json) +- [IVFPQ, 65,536 rows](results/index-build-memory-20261118-pq-65536.json) + ## Staged maintainer decision 1. Review the completed red/green checks and four bounded studies, including the poor synthetic PQ overlap. Reproduce on another machine and measure a duplicate-free fixture and representative data before setting any resource budget or performance target. Inspect dedup disk/CPU cost at larger volumes. -2. Decide the dedup identity/bounds separately with PR #140. This study preserves - current behavior, including known collisions. Compare disk-backed dedup cost - against alternatives before choosing a production implementation. Consider a - matched-training-sample experiment to isolate sampling-induced ANN variation. +2. Preserve the current production dedup identity and range rejection from #203. + Compare the disk-backed composite-key storage/CPU cost against alternatives + before choosing a production implementation. Consider a matched-training- + sample experiment to isolate sampling-induced ANN variation. 3. Only after measurements, propose an explicit opt-in production builder with supported resource budgets, input validation, snapshot/locking policy, disk requirements and publication/recovery guarantees. Keep defaults until reviewed. @@ -290,17 +338,19 @@ neighbor agreement is not evidence of semantic-retrieval quality. **Body:** IVF/PQ builds currently merge the complete raw float32 corpus before batched addition. Add an isolated two-pass prototype and bounded synthetic -benchmark using disk-backed legacy dedup, reservoir training, ordered addition -and compatible streamed metadata. Production defaults remain unchanged. - -Validation: reproduce the legacy allocation failure; 47 focused/neighboring tests -pass with no skips; Ruff and whitespace checks pass. Four configurations include -three fresh-process pairs each. At 65,536 × 256 rows, median RSS was -268.62 → 142.80 MiB (IVFFlat) and 263.98 → 74.48 MiB (PQ); raw reports include -wall time, counts, metadata, sizes and neighbor overlap. Limits: final index -memory remains, sampling changes ANN results, and random-data PQ overlap is poor. -The prototype requires immutable shards and a new output directory; Linux, -production-scale recall and crash recovery remain unvalidated. +benchmark using disk-backed tuple dedup compatible with the current range-derived +identity, reservoir training, ordered addition and compatible streamed metadata. +Production defaults remain unchanged. + +Validation on current upstream: 48 focused/neighboring tests pass with no skips; +four benchmark configurations have three fresh-process pairs each and stored +raw reports. At 65,536 × 256 rows, median RSS was 272.73 → 144.52 MiB (IVFFlat) +and 264.75 → 76.23 MiB (PQ); metadata ordering and dtypes match the current +production merge. Ruff could not be run locally because its executable is +absent and package download failed on DNS; whitespace checks pass. Limits: final +index memory remains, sampling changes ANN results, and random-data PQ overlap +is poor. The prototype requires immutable shards and a new output directory; +Linux, production-scale recall and crash recovery remain unvalidated. ## Maintainer discussion draft (not posted) @@ -311,9 +361,10 @@ in memory. SQLite moves dedup state to disk with a bounded page cache, but adds per-row SQL work and scratch-space requirements that need measurement at scale. I suggest keeping this as a benchmark/proposal until those costs are understood. -Two policy decisions should stay separate. This study retains the packed-int64 -key semantics, including collisions/overflow, so it does not absorb #140. -Reservoir sampling is reproducible for a fixed seed but changes trained +The fixed-decimal collision bug discussed in #140 was fixed upstream in #203. +This rebase adopts that tuple identity and its `int64` range rejection. The +remaining question is whether the SQLite composite-key cost is acceptable for +an opt-in builder. Reservoir sampling is reproducible for a fixed seed but changes trained centroids and ANN neighbors compared with the current sampling method. Is that acceptable for an explicitly opt-in rebuild, or should a matched-training-sample experiment be required first? Neither current overlap results nor the small @@ -333,15 +384,14 @@ separately. No production default change is proposed by this contribution. `_merge_all_shards` allocates the complete raw float32 corpus before IVF/PQ or Qdrant batching. PQ (#155) reduces final index storage but does not remove this -input allocation. Could we first review a bounded synthetic memory study and -an opt-in two-pass prototype before changing defaults? The draft compares fresh -processes on peak RSS, build time, output size, metadata order and neighbor -overlap, preserving existing packed-key dedup pending #140. On a 65,536 × 256 -synthetic fixture, three-repeat median RSS was 268.62 → 142.80 MiB for IVFFlat -and 263.98 → 74.48 MiB for PQ. Both retained 62,260 rows with identical metadata; -IVFFlat matched exact search when probing all lists, but ANN neighbors differ -and aggressive PQ has poor overlap on random data. These laptop measurements -are not production-scale or semantic-retrieval claims. Decisions needed include -dedup storage cost, shard/sample limits, publication and recovery, and whether -final IVFFlat memory requires a separate external-memory design. This proposal -does not include the #140 collision fix or a production backend rewrite. +input allocation. Could we review a bounded synthetic memory study and an +opt-in two-pass prototype before changing defaults? The 2026-11-18 rebase +preserves current tuple dedup identity and range rejection from #203. On the +65,536 × 256 synthetic fixture, three-repeat median RSS was 272.73 → 144.52 MiB +for IVFFlat and 264.75 → 76.23 MiB for PQ; both retained 62,260 rows with all +five metadata arrays aligned. IVFFlat matched exact search when probing all +lists, but ANN neighbors differ and aggressive PQ has poor overlap on random +data. These laptop measurements are not production-scale or semantic-retrieval +claims. Decisions still include dedup disk/CPU cost, shard/sample limits, +publication and recovery, and whether final IVFFlat memory requires a separate +external-memory design. This does not propose a production backend rewrite. diff --git a/benchmarks/results/index-build-memory-20261118-flat-32768.json b/benchmarks/results/index-build-memory-20261118-flat-32768.json new file mode 100644 index 0000000..4d6ff06 --- /dev/null +++ b/benchmarks/results/index-build-memory-20261118-flat-32768.json @@ -0,0 +1,142 @@ +{ + "configuration": { + "work_dir": "/tmp/pixelrag-index-build-memory-20261118-flat-32768", + "rows": 32768, + "dim": 256, + "shard_rows": 4096, + "train_sample": 8192, + "batch_size": 1024, + "nlist": 64, + "nprobe": 8, + "pq_m": 0, + "threads": 1, + "seed": 2026, + "repeats": 3, + "worker": null, + "output": null + }, + "platform": "macOS-26.5.1-arm64-arm-64bit", + "python": "3.12.13 (main, Mar 3 2026, 12:39:30) [Clang 21.0.0 (clang-2100.0.123.102)]", + "numpy": "2.4.5", + "faiss": "1.14.2", + "cpu_count": 10, + "runs": [ + { + "metrics": { + "legacy": { + "builder": "legacy", + "build_wall_seconds": 0.252680584002519, + "peak_rss_raw": 180928512, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 172.546875, + "index_bytes": 32192347, + "metadata_bytes": 748416, + "total_output_bytes": 32941131 + }, + "prototype": { + "builder": "prototype", + "build_wall_seconds": 0.228142084000865, + "peak_rss_raw": 114229248, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 108.9375, + "index_bytes": 32192347, + "metadata_bytes": 748416, + "total_output_bytes": 32941202 + } + }, + "validation": { + "legacy": { + "vector_count": 31130, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.27421875, + "all_lists_overlap_at_10_with_exact": 1.0 + }, + "prototype": { + "vector_count": 31130, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.26640625, + "all_lists_overlap_at_10_with_exact": 1.0 + }, + "between_builders_ann_overlap_at_10": 0.22734375 + } + }, + { + "metrics": { + "prototype": { + "builder": "prototype", + "build_wall_seconds": 0.24461191600130405, + "peak_rss_raw": 117194752, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 111.765625, + "index_bytes": 32192347, + "metadata_bytes": 748416, + "total_output_bytes": 32941202 + }, + "legacy": { + "builder": "legacy", + "build_wall_seconds": 0.24357783299637958, + "peak_rss_raw": 181108736, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 172.71875, + "index_bytes": 32192347, + "metadata_bytes": 748416, + "total_output_bytes": 32941131 + } + }, + "validation": { + "prototype": { + "vector_count": 31130, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.26640625, + "all_lists_overlap_at_10_with_exact": 1.0 + }, + "legacy": { + "vector_count": 31130, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.27421875, + "all_lists_overlap_at_10_with_exact": 1.0 + }, + "between_builders_ann_overlap_at_10": 0.22734375 + } + }, + { + "metrics": { + "legacy": { + "builder": "legacy", + "build_wall_seconds": 0.26885612499609124, + "peak_rss_raw": 181813248, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 173.390625, + "index_bytes": 32192347, + "metadata_bytes": 748416, + "total_output_bytes": 32941131 + }, + "prototype": { + "builder": "prototype", + "build_wall_seconds": 0.27017641699058004, + "peak_rss_raw": 114900992, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 109.578125, + "index_bytes": 32192347, + "metadata_bytes": 748416, + "total_output_bytes": 32941202 + } + }, + "validation": { + "legacy": { + "vector_count": 31130, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.27421875, + "all_lists_overlap_at_10_with_exact": 1.0 + }, + "prototype": { + "vector_count": 31130, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.26640625, + "all_lists_overlap_at_10_with_exact": 1.0 + }, + "between_builders_ann_overlap_at_10": 0.22734375 + } + } + ] +} diff --git a/benchmarks/results/index-build-memory-20261118-flat-65536.json b/benchmarks/results/index-build-memory-20261118-flat-65536.json new file mode 100644 index 0000000..a6dcd0d --- /dev/null +++ b/benchmarks/results/index-build-memory-20261118-flat-65536.json @@ -0,0 +1,142 @@ +{ + "configuration": { + "work_dir": "/tmp/pixelrag-index-build-memory-20261118-flat-65536", + "rows": 65536, + "dim": 256, + "shard_rows": 4096, + "train_sample": 8192, + "batch_size": 1024, + "nlist": 64, + "nprobe": 8, + "pq_m": 0, + "threads": 1, + "seed": 2026, + "repeats": 3, + "worker": null, + "output": null + }, + "platform": "macOS-26.5.1-arm64-arm-64bit", + "python": "3.12.13 (main, Mar 3 2026, 12:39:30) [Clang 21.0.0 (clang-2100.0.123.102)]", + "numpy": "2.4.5", + "faiss": "1.14.2", + "cpu_count": 10, + "runs": [ + { + "metrics": { + "legacy": { + "builder": "legacy", + "build_wall_seconds": 0.5662763750005979, + "peak_rss_raw": 284852224, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 271.65625, + "index_bytes": 64318507, + "metadata_bytes": 1495536, + "total_output_bytes": 65814411 + }, + "prototype": { + "builder": "prototype", + "build_wall_seconds": 0.500621625003987, + "peak_rss_raw": 150421504, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 143.453125, + "index_bytes": 64318507, + "metadata_bytes": 1495536, + "total_output_bytes": 65814482 + } + }, + "validation": { + "legacy": { + "vector_count": 62260, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.27890625, + "all_lists_overlap_at_10_with_exact": 1.0 + }, + "prototype": { + "vector_count": 62260, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.28750000000000003, + "all_lists_overlap_at_10_with_exact": 1.0 + }, + "between_builders_ann_overlap_at_10": 0.228125 + } + }, + { + "metrics": { + "prototype": { + "builder": "prototype", + "build_wall_seconds": 0.528344875012408, + "peak_rss_raw": 155877376, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 148.65625, + "index_bytes": 64318507, + "metadata_bytes": 1495536, + "total_output_bytes": 65814482 + }, + "legacy": { + "builder": "legacy", + "build_wall_seconds": 0.5614774170098826, + "peak_rss_raw": 285982720, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 272.734375, + "index_bytes": 64318507, + "metadata_bytes": 1495536, + "total_output_bytes": 65814411 + } + }, + "validation": { + "prototype": { + "vector_count": 62260, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.28750000000000003, + "all_lists_overlap_at_10_with_exact": 1.0 + }, + "legacy": { + "vector_count": 62260, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.27890625, + "all_lists_overlap_at_10_with_exact": 1.0 + }, + "between_builders_ann_overlap_at_10": 0.228125 + } + }, + { + "metrics": { + "legacy": { + "builder": "legacy", + "build_wall_seconds": 0.5042654999997467, + "peak_rss_raw": 289718272, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 276.296875, + "index_bytes": 64318507, + "metadata_bytes": 1495536, + "total_output_bytes": 65814411 + }, + "prototype": { + "builder": "prototype", + "build_wall_seconds": 0.48354583299078513, + "peak_rss_raw": 151535616, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 144.515625, + "index_bytes": 64318507, + "metadata_bytes": 1495536, + "total_output_bytes": 65814482 + } + }, + "validation": { + "legacy": { + "vector_count": 62260, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.27890625, + "all_lists_overlap_at_10_with_exact": 1.0 + }, + "prototype": { + "vector_count": 62260, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.28750000000000003, + "all_lists_overlap_at_10_with_exact": 1.0 + }, + "between_builders_ann_overlap_at_10": 0.228125 + } + } + ] +} diff --git a/benchmarks/results/index-build-memory-20261118-pq-32768.json b/benchmarks/results/index-build-memory-20261118-pq-32768.json new file mode 100644 index 0000000..e77e4de --- /dev/null +++ b/benchmarks/results/index-build-memory-20261118-pq-32768.json @@ -0,0 +1,142 @@ +{ + "configuration": { + "work_dir": "/tmp/pixelrag-index-build-memory-20261118-pq-32768", + "rows": 32768, + "dim": 256, + "shard_rows": 4096, + "train_sample": 8192, + "batch_size": 1024, + "nlist": 64, + "nprobe": 8, + "pq_m": 16, + "threads": 1, + "seed": 2026, + "repeats": 3, + "worker": null, + "output": null + }, + "platform": "macOS-26.5.1-arm64-arm-64bit", + "python": "3.12.13 (main, Mar 3 2026, 12:39:30) [Clang 21.0.0 (clang-2100.0.123.102)]", + "numpy": "2.4.5", + "faiss": "1.14.2", + "cpu_count": 10, + "runs": [ + { + "metrics": { + "legacy": { + "builder": "legacy", + "build_wall_seconds": 0.2881904579990078, + "peak_rss_raw": 173948928, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 165.890625, + "index_bytes": 580692, + "metadata_bytes": 748416, + "total_output_bytes": 1329468 + }, + "prototype": { + "builder": "prototype", + "build_wall_seconds": 0.2944977500010282, + "peak_rss_raw": 79101952, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 75.4375, + "index_bytes": 580692, + "metadata_bytes": 748416, + "total_output_bytes": 1329539 + } + }, + "validation": { + "legacy": { + "vector_count": 31130, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.00859375, + "all_lists_overlap_at_10_with_exact": 0.00859375 + }, + "prototype": { + "vector_count": 31130, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.00546875, + "all_lists_overlap_at_10_with_exact": 0.00546875 + }, + "between_builders_ann_overlap_at_10": 0.0 + } + }, + { + "metrics": { + "prototype": { + "builder": "prototype", + "build_wall_seconds": 0.28643008299695794, + "peak_rss_raw": 79151104, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 75.484375, + "index_bytes": 580692, + "metadata_bytes": 748416, + "total_output_bytes": 1329539 + }, + "legacy": { + "builder": "legacy", + "build_wall_seconds": 0.29893383300805, + "peak_rss_raw": 173916160, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 165.859375, + "index_bytes": 580692, + "metadata_bytes": 748416, + "total_output_bytes": 1329468 + } + }, + "validation": { + "prototype": { + "vector_count": 31130, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.00546875, + "all_lists_overlap_at_10_with_exact": 0.00546875 + }, + "legacy": { + "vector_count": 31130, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.00859375, + "all_lists_overlap_at_10_with_exact": 0.00859375 + }, + "between_builders_ann_overlap_at_10": 0.0 + } + }, + { + "metrics": { + "legacy": { + "builder": "legacy", + "build_wall_seconds": 0.2860917499929201, + "peak_rss_raw": 174096384, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 166.03125, + "index_bytes": 580692, + "metadata_bytes": 748416, + "total_output_bytes": 1329468 + }, + "prototype": { + "builder": "prototype", + "build_wall_seconds": 0.267969708002056, + "peak_rss_raw": 78168064, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 74.546875, + "index_bytes": 580692, + "metadata_bytes": 748416, + "total_output_bytes": 1329539 + } + }, + "validation": { + "legacy": { + "vector_count": 31130, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.00859375, + "all_lists_overlap_at_10_with_exact": 0.00859375 + }, + "prototype": { + "vector_count": 31130, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.00546875, + "all_lists_overlap_at_10_with_exact": 0.00546875 + }, + "between_builders_ann_overlap_at_10": 0.0 + } + } + ] +} diff --git a/benchmarks/results/index-build-memory-20261118-pq-65536.json b/benchmarks/results/index-build-memory-20261118-pq-65536.json new file mode 100644 index 0000000..4fcc0ec --- /dev/null +++ b/benchmarks/results/index-build-memory-20261118-pq-65536.json @@ -0,0 +1,142 @@ +{ + "configuration": { + "work_dir": "/tmp/pixelrag-index-build-memory-20261118-pq-65536", + "rows": 65536, + "dim": 256, + "shard_rows": 4096, + "train_sample": 8192, + "batch_size": 1024, + "nlist": 64, + "nprobe": 8, + "pq_m": 16, + "threads": 1, + "seed": 2026, + "repeats": 3, + "worker": null, + "output": null + }, + "platform": "macOS-26.5.1-arm64-arm-64bit", + "python": "3.12.13 (main, Mar 3 2026, 12:39:30) [Clang 21.0.0 (clang-2100.0.123.102)]", + "numpy": "2.4.5", + "faiss": "1.14.2", + "cpu_count": 10, + "runs": [ + { + "metrics": { + "legacy": { + "builder": "legacy", + "build_wall_seconds": 0.5200417500018375, + "peak_rss_raw": 277118976, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 264.28125, + "index_bytes": 1078772, + "metadata_bytes": 1495536, + "total_output_bytes": 2574668 + }, + "prototype": { + "builder": "prototype", + "build_wall_seconds": 0.4908812920039054, + "peak_rss_raw": 80887808, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 77.140625, + "index_bytes": 1078772, + "metadata_bytes": 1495536, + "total_output_bytes": 2574739 + } + }, + "validation": { + "legacy": { + "vector_count": 62260, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.01015625, + "all_lists_overlap_at_10_with_exact": 0.0109375 + }, + "prototype": { + "vector_count": 62260, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.00625, + "all_lists_overlap_at_10_with_exact": 0.00625 + }, + "between_builders_ann_overlap_at_10": 0.0015625 + } + }, + { + "metrics": { + "prototype": { + "builder": "prototype", + "build_wall_seconds": 0.46480866700585466, + "peak_rss_raw": 79364096, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 75.6875, + "index_bytes": 1078772, + "metadata_bytes": 1495536, + "total_output_bytes": 2574739 + }, + "legacy": { + "builder": "legacy", + "build_wall_seconds": 0.531655957995099, + "peak_rss_raw": 278085632, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 265.203125, + "index_bytes": 1078772, + "metadata_bytes": 1495536, + "total_output_bytes": 2574668 + } + }, + "validation": { + "prototype": { + "vector_count": 62260, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.00625, + "all_lists_overlap_at_10_with_exact": 0.00625 + }, + "legacy": { + "vector_count": 62260, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.01015625, + "all_lists_overlap_at_10_with_exact": 0.0109375 + }, + "between_builders_ann_overlap_at_10": 0.0015625 + } + }, + { + "metrics": { + "legacy": { + "builder": "legacy", + "build_wall_seconds": 0.5291202500084182, + "peak_rss_raw": 277610496, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 264.75, + "index_bytes": 1078772, + "metadata_bytes": 1495536, + "total_output_bytes": 2574668 + }, + "prototype": { + "builder": "prototype", + "build_wall_seconds": 0.46531050000339746, + "peak_rss_raw": 79937536, + "peak_rss_os_unit": "bytes", + "peak_rss_mib": 76.234375, + "index_bytes": 1078772, + "metadata_bytes": 1495536, + "total_output_bytes": 2574739 + } + }, + "validation": { + "legacy": { + "vector_count": 62260, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.01015625, + "all_lists_overlap_at_10_with_exact": 0.0109375 + }, + "prototype": { + "vector_count": 62260, + "metadata_alignment": "all five arrays and dtypes equal legacy merge", + "ann_overlap_at_10_with_exact": 0.00625, + "all_lists_overlap_at_10_with_exact": 0.00625 + }, + "between_builders_ann_overlap_at_10": 0.0015625 + } + } + ] +} diff --git a/benchmarks/streaming_ivf.py b/benchmarks/streaming_ivf.py index 0f94fb1..3f6f4c9 100644 --- a/benchmarks/streaming_ivf.py +++ b/benchmarks/streaming_ivf.py @@ -6,6 +6,7 @@ """ import json +import math import os import shutil import sqlite3 @@ -84,11 +85,13 @@ def build_streaming_ivf( pq_m=0, pq_nbits=8, ): - """Two passes: first-wins dedup + reservoir training, then streaming add. + """First-wins tuple dedup + reservoir training, then a streaming add pass. - Retains legacy *packed int64* collisions and overflow, intentionally. This - is not the collision fix discussed in PR #140. SQLite stores signed keys; - survivor positions are disk files, not a corpus-sized Python set or list. + SQLite's composite key implements the same tuple identity as the current + production _dedup_keys radix packer without allocating a corpus-sized key + array. The full observed ranges are still checked against the production + int64-key limit before training. Survivor positions are disk files, not a + corpus-sized Python set or list. Destination must not exist. Publication is a same-filesystem directory rename; concurrent builders targeting the same destination are unsupported. """ @@ -114,7 +117,17 @@ def build_streaming_ivf( try: db.execute("PRAGMA cache_size=-2048") # 2 MiB page cache db.execute("PRAGMA temp_store=FILE") - db.execute("CREATE TABLE seen (key INTEGER PRIMARY KEY)") + db.execute( + "CREATE TABLE seen (" + "article_id INTEGER NOT NULL, " + "tile_index INTEGER NOT NULL, " + "chunk_index INTEGER NOT NULL, " + "PRIMARY KEY (article_id, tile_index, chunk_index)" + ") WITHOUT ROWID" + ) + identity_fields = ("article_ids", "tile_indices", "chunk_indices") + lows = [None] * len(identity_fields) + highs = [None] * len(identity_fields) for shard_id, path in enumerate(files): vectors, metadata = _read_shard(path, dim) if dim is None: @@ -122,16 +135,24 @@ def build_streaming_ivf( if pq_m and dim % pq_m: raise ValueError("pq_m must divide dimension evenly") reservoir = np.empty((train_sample, dim), dtype=np.float32) - # Match legacy casts before arithmetic, including int64 wrap. - keys = ( - metadata["article_ids"] * 100_000_000 - + metadata["tile_indices"].astype(np.int64) * 10_000 - + metadata["chunk_indices"].astype(np.int64) - ) + for field_id, name in enumerate(identity_fields): + values = metadata[name] + if values.size: + low, high = int(values.min()), int(values.max()) + lows[field_id] = ( + low if lows[field_id] is None else min(lows[field_id], low) + ) + highs[field_id] = ( + high + if highs[field_id] is None + else max(highs[field_id], high) + ) selected = [] # bounded by one shard, not the corpus - for row, key in enumerate(keys): + identity_arrays = tuple(metadata[name] for name in identity_fields) + for row, identity in enumerate(zip(*identity_arrays)): cursor = db.execute( - "INSERT OR IGNORE INTO seen VALUES (?)", (int(key),) + "INSERT OR IGNORE INTO seen VALUES (?, ?, ?)", + tuple(int(value) for value in identity), ) if not cursor.rowcount: continue @@ -145,11 +166,17 @@ def build_streaming_ivf( scratch / f"rows_{shard_id}.npy", np.asarray(selected, dtype=np.int64), ) - del vectors, metadata, keys, selected + del vectors, metadata, identity_arrays, selected finally: db.close() if count == 0: raise ValueError("No vectors after deduplication") + spans = [high - low + 1 for low, high in zip(lows, highs)] + if math.prod(spans) - 1 > np.iinfo(np.int64).max: + raise ValueError( + "(article_id, tile_index, chunk_index) ranges are too wide to pack " + f"into an int64 dedup key: spans {spans}" + ) metric_type = faiss.METRIC_INNER_PRODUCT if metric == "ip" else faiss.METRIC_L2 quantizer = faiss.IndexFlatIP(dim) if metric == "ip" else faiss.IndexFlatL2(dim) nlist = min(nlist, count) diff --git a/tests/test_index_build_memory.py b/tests/test_index_build_memory.py index fd01c6e..cb65958 100644 --- a/tests/test_index_build_memory.py +++ b/tests/test_index_build_memory.py @@ -94,25 +94,37 @@ def test_metadata_order_and_neighbor_alignment(shards, tmp_path, pq_m, metric): np.testing.assert_array_equal(ids, exact.search(queries, 5)[1]) -def test_legacy_collisions_overflow_and_first_wins_are_preserved(tmp_path): +def test_current_tuple_identity_and_first_wins_are_preserved(tmp_path): source = tmp_path / "shards" source.mkdir() - # (1,10000,0) collides with (2,0,0); (3,0,10000) with (3,1,0). - # Large article IDs exercise the current int64 arithmetic overflow too. + # These three distinct pairs collided under the old fixed-decimal key: + # (1,10000,0)/(2,0,0), (3,0,10000)/(3,1,0), and + # (2,-1,0)/(1,9999,0). A repeated tuple in shard 1 must keep its first row. + identities = [ + (1, 10000, 0), + (2, 0, 0), + (3, 0, 10000), + (3, 1, 0), + (2, -1, 0), + (1, 9999, 0), + ] for shard in range(2): + rows = identities if shard == 0 else [*identities, identities[0]] np.savez_compressed( source / f"shard_{shard}.npz", - embeddings=np.arange(24, dtype=np.float32).reshape(6, 4) + shard, - article_ids=np.array([1, 2, 3, 3, 2**62, 2**62], dtype=np.int64), - tile_indices=np.array([10000, 0, 0, 1, 0, 0], dtype=np.int32), - chunk_indices=np.array([0, 0, 10000, 0, 0, 0], dtype=np.int32), - y_offsets=np.arange(6, dtype=np.int32) + shard * 100, - tile_heights=np.full(6, 100 + shard, dtype=np.int32), + embeddings=np.arange(len(rows) * 4, dtype=np.float32).reshape(-1, 4) + + shard * 100, + article_ids=np.array([row[0] for row in rows], dtype=np.int64), + tile_indices=np.array([row[1] for row in rows], dtype=np.int32), + chunk_indices=np.array([row[2] for row in rows], dtype=np.int32), + y_offsets=np.arange(len(rows), dtype=np.int32) + shard * 100, + tile_heights=np.full(len(rows), 100 + shard, dtype=np.int32), ) expected = _merge_all_shards(sorted(source.glob("*.npz"))) build_streaming_ivf(source, tmp_path / "out", nlist=1, train_sample=3) index = faiss.read_index(str(tmp_path / "out/index.faiss")) index.make_direct_map() + assert index.ntotal == len(identities) with np.load(tmp_path / "out/metadata.npz") as actual: for name in FIELDS: np.testing.assert_array_equal(actual[name], expected[name]) @@ -122,6 +134,25 @@ def test_legacy_collisions_overflow_and_first_wins_are_preserved(tmp_path): ) +def test_current_int64_key_range_limit_is_preserved(tmp_path): + source = tmp_path / "shards" + source.mkdir() + np.savez_compressed( + source / "shard_000000.npz", + embeddings=np.arange(8, dtype=np.float32).reshape(2, 4), + article_ids=np.array([0, 2**62], dtype=np.int64), + tile_indices=np.array([0, 10000], dtype=np.int32), + chunk_indices=np.array([0, 0], dtype=np.int32), + y_offsets=np.arange(2, dtype=np.int32), + tile_heights=np.full(2, 100, dtype=np.int32), + ) + with pytest.raises(ValueError, match="ranges are too wide to pack"): + _merge_all_shards(sorted(source.glob("*.npz"))) + with pytest.raises(ValueError, match="ranges are too wide to pack"): + build_streaming_ivf(source, tmp_path / "out", nlist=1, train_sample=2) + assert not (tmp_path / "out").exists() + + def test_write_failure_does_not_publish_partial_output(shards, tmp_path, monkeypatch): def fail_write(index, path): Path(path).write_bytes(b"partial")