pyproject.toml

[project]
name = "llama-extract-bench"
dynamic = ["version"]
description = "ExtractBench: a benchmark and evaluation harness for schema-guided document extraction on real-world enterprise documents"
readme = "README.md"
license = "Apache-2.0"
license-files = ["LICENSE"]
requires-python = ">=3.12"
authors = [
    { name = "LlamaIndex" },
]
keywords = ["document-extraction", "structured-extraction", "json-schema", "benchmark", "evaluation", "pdf", "grounding", "citations"]
classifiers = [
    "Development Status :: 4 - Beta",
    "Intended Audience :: Developers",
    "Intended Audience :: Science/Research",
    "License :: OSI Approved :: Apache Software License",
    "Operating System :: OS Independent",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: 3.13",
    "Topic :: Scientific/Engineering :: Artificial Intelligence",
    "Topic :: Text Processing",
]
dependencies = [
    "aiohttp>=3.9.0",
    "anls-star>=0.1.0",
    "apted>=1.0.3",
    "autoevals>=0.0.20",
    "beautifulsoup4>=4.12.0",
    "bleach>=6.0.0",
    "fire>=0.7.1",
    "fuzzysearch>=0.7.3",
    "huggingface-hub>=0.20.0",
    "lxml>=5.0.0",
    "markdown>=3.0",
    "markdown2>=2.4.0",
    "numpy>=1.24.0",
    "pandas>=2.0.0",
    # Owns the core shared with parse-bench (see the re-export modules under
    # schemas/). The floor is 1.0.2 because that is what the internal harness
    # pins (a 1.0.2 rev carrying the v2.1 contract backport); every re-exported
    # module is identical from 1.0.2 through 1.0.4.
    "parse-bench>=1.0.2",
    "pydantic>=2.0.0",
    "python-dateutil>=2.9.0",
    "python-dotenv>=1.0.0",
    "python-Levenshtein>=0.25.0",
    "rapidfuzz>=3.0.0",
    "requests>=2.31.0",
    "rich>=13.7.0",
    "scikit-learn>=1.8.0",
    "scipy>=1.16.3",
    "unidecode>=1.3.0",
]

[project.optional-dependencies]
# Every provider SDK. Per-provider extras below are for users who only need one
# system; `runners` is their union.
runners = [
    "amazon-textract-textractor>=1.7.0",
    "anthropic>=0.77.1",
    "azure-ai-documentintelligence>=1.0.0",
    "boto3>=1.34.0",
    "chunkr-ai>=0.0.43",
    "datalab-python-sdk",
    "docling-core>=2.71.0",
    "extend-ai>=1.16.0",
    "google-cloud-documentai>=2.20.0",
    "google-genai>=1.0.0",
    "httpx>=0.28.0",
    "infinity-parser2>=0.3.0",
    "landingai-ade>=1.4.0",
    "llama-cloud>=1.4.1",
    "openai>=1.0.0",
    "pdf2image>=1.16.0",
    "pillow>=10.0.0",
    "pymupdf>=1.24.0",
    "pypdf>=6.4.0",
    "pytesseract>=0.3.10",
    "reductoai>=0.13.0",
    "unstructured-client>=0.26.0",
    "warp-ingest[ocr]>=2.0.1",
]
llamaextract = ["llama-cloud>=1.4.1", "httpx>=0.28.0"]
openai = ["openai>=1.0.0", "pdf2image>=1.16.0", "pillow>=10.0.0", "pymupdf>=1.24.0", "pypdf>=6.4.0"]
anthropic = ["anthropic>=0.77.1", "pdf2image>=1.16.0", "pillow>=10.0.0", "pymupdf>=1.24.0", "pypdf>=6.4.0"]
google = ["google-genai>=1.0.0", "pdf2image>=1.16.0", "pillow>=10.0.0", "pymupdf>=1.24.0", "pypdf>=6.4.0"]
# Self-hosted open-weight models served over an OpenAI-compatible endpoint (vLLM).
openweights = ["openai>=1.0.0", "pdf2image>=1.16.0", "pillow>=10.0.0", "pymupdf>=1.24.0", "pypdf>=6.4.0"]
reducto = ["reductoai>=0.13.0"]
extend = ["extend-ai>=1.16.0"]
datalab = ["datalab-python-sdk"]
landingai = ["landingai-ade>=1.4.0"]
azure = ["azure-ai-documentintelligence>=1.0.0", "httpx>=0.28.0"]
aws = ["boto3>=1.34.0", "amazon-textract-textractor>=1.7.0"]
local = ["pymupdf>=1.24.0", "pypdf>=6.4.0", "pytesseract>=0.3.10", "pdf2image>=1.16.0", "pillow>=10.0.0"]
fast = [
    "numba>=0.59.0",
]
dev = [
    "mypy>=1.18.2",
    "numba>=0.59.0",
    "pytest>=8.0.0",
    "ruff>=0.14.5",
]

[project.urls]
Homepage = "https://www.extractbench.ai"
Repository = "https://github.com/run-llama/ExtractBench"
Issues = "https://github.com/run-llama/ExtractBench/issues"
Dataset = "https://huggingface.co/datasets/llamaindex/ExtractBench"
Paper = "https://arxiv.org/abs/2607.29677"
Changelog = "https://github.com/run-llama/ExtractBench/blob/main/CHANGELOG.md"

[project.scripts]
extract-bench = "extract_bench.cli:main"

[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"

[tool.hatch.version]
path = "src/extract_bench/__init__.py"

[tool.hatch.build.targets.wheel]
# ``packages`` already carries analysis/static into the wheel; a force-include
# for the same directory makes hatchling add every asset twice and abort.
packages = ["src/extract_bench"]

[tool.hatch.build.targets.sdist]
only-include = [
    "src/extract_bench",
    "tests",
    "README.md",
    "LICENSE",
    "CHANGELOG.md",
]
exclude = [
    "**/__pycache__",
    "**/.gitkeep",
]

[tool.ruff]
line-length = 120
target-version = "py312"

[tool.ruff.lint]
select = [
    "E",  # pycodestyle errors
    "W",  # pycodestyle warnings
    "F",  # pyflakes
    "I",  # isort
    "B",  # flake8-bugbear
    "C4", # flake8-comprehensions
    "UP", # pyupgrade
]

[tool.ruff.lint.per-file-ignores]
"src/extract_bench/analysis/detailed_report.py" = ["E501", "E402"]
"src/extract_bench/analysis/comparison_report.py" = ["E501"]
# Vendored/community-submitted parse providers keep their upstream style.
"src/extract_bench/inference/providers/parse/kdl_frontier_nano.py" = ["E402", "E501", "B905", "B904"]
"src/extract_bench/inference/providers/parse/liteparse.py" = ["E402"]
"src/extract_bench/inference/providers/parse/infinity_parser2.py" = ["B905"]
# StrEnum migration deferred to keep this vendored metric module byte-stable.
"src/extract_bench/evaluation/metrics/extract/test_types.py" = ["UP042"]
# Benchmark closures are invoked within the same loop iteration, so the
# late-binding B023 warning is a false positive here.
"scripts/bench_table_metrics.py" = ["B023"]

[tool.ruff.lint.isort]
known-first-party = ["extract_bench"]

[tool.mypy]
python_version = "3.12"
warn_return_any = true
warn_unused_configs = true

[[tool.mypy.overrides]]
module = [
    "apted.*",
    "apted",
    "autoevals.number",
    "autoevals.string",
    "bleach",
    "boto3",
    "botocore.*",
    "datalab_sdk.*",
    "datalab_sdk",
    "dateutil.*",
    "dateutil",
    "fire",
    "fitz",
    "fuzzysearch",
    "lxml.*",
    "markdown2",
    "pandas",
    "pytesseract",
    "scipy.*",
    "sklearn.*",
    "textractor.*",
    "textractor",
]
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "extract_bench.evaluation.metrics.parse._vendor_grits_reference"
ignore_errors = true

[[tool.mypy.overrides]]
module = "extract_bench.test_cases.parse_rule_schemas"
ignore_errors = true

[tool.pytest.ini_options]
testpaths = ["tests"]
python_files = ["test_*.py"]
python_classes = ["Test*"]
python_functions = ["test_*"]
addopts = ["--verbose", "--strict-markers", "--tb=short", "-rs"]