pyproject.toml

[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"

[project]
name = "crawlee"
version = "0.6.8"
description = "Crawlee for Python"
authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
license = { file = "LICENSE" }
readme = "README.md"
requires-python = ">=3.9"
classifiers = [
    "Development Status :: 4 - Beta",
    "Intended Audience :: Developers",
    "License :: OSI Approved :: Apache Software License",
    "Operating System :: OS Independent",
    "Programming Language :: Python :: 3.9",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: 3.13",
    "Topic :: Software Development :: Libraries",
]
keywords = [
    "apify",
    "automation",
    "chrome",
    "crawlee",
    "crawler",
    "headless",
    "scraper",
    "scraping",
]
dependencies = [
    "apify_fingerprint_datapoints>=0.0.2",
    "browserforge>=1.2.3",
    "cachetools>=5.5.0",
    "colorama>=0.4.0",
    "eval-type-backport>=0.2.0",
    "httpx[brotli,http2,zstd]>=0.27.0",
    "more-itertools>=10.2.0",
    "protego>=0.4.0",
    "psutil>=6.0.0",
    "pydantic-settings>=2.2.0,<2.7.0",
    "pydantic>=2.8.0,!=2.10.0,!=2.10.1,!=2.10.2",
    "pyee>=9.0.0",
    "sortedcollections>=2.1.0",
    "sortedcontainers>=2.4.0",
    "tldextract>=5.1.0",
    "typing-extensions>=4.1.0",
    "yarl>=1.18.0",
]

[project.optional-dependencies]
# TODO: automate the generation of all extra
# https://github.com/apify/crawlee-python/issues/995
all = [
    "beautifulsoup4[lxml]>=4.12.0",
    "cookiecutter>=2.6.0",
    "curl-cffi>=0.9.0",
    "html5lib>=1.0",
    "inquirer>=3.3.0",
    "jaro-winkler>=2.0.3",
    "parsel>=1.10.0",
    "playwright>=1.27.0",
    "scikit-learn==1.5.2; python_version == '3.9'",
    "scikit-learn>=1.6.0; python_version >= '3.10'",
    'typer>=0.12.0',
]
adaptive-crawler = [
    "jaro-winkler>=2.0.3",
    "playwright>=1.27.0",
    "scikit-learn==1.5.2; python_version == '3.9'",
    "scikit-learn>=1.6.0; python_version >= '3.10'",
]
beautifulsoup = ["beautifulsoup4[lxml]>=4.12.0", "html5lib>=1.0"]
cli = ["cookiecutter>=2.6.0", "inquirer>=3.3.0", "rich>=13.9.0", "typer>=0.12.0"]
curl-impersonate = ["curl-cffi>=0.9.0"]
parsel = ["parsel>=1.10.0"]
playwright = ["playwright>=1.27.0"]

[project.scripts]
crawlee = "crawlee._cli:cli"

[project.urls]
"Homepage" = "https://crawlee.dev/python"
"Apify homepage" = "https://apify.com"
"Changelog" = "https://crawlee.dev/python/docs/changelog"
"Documentation" = "https://crawlee.dev/python/docs/"
"Issue tracker" = "https://github.com/apify/crawlee-python/issues"
"Repository" = "https://github.com/apify/crawlee-python"

[dependency-groups]
dev = [
    "apify_client",  # For e2e tests.
    "build~=1.2.2",  # For e2e tests.
    "mypy~=1.15.0",
    "pre-commit~=4.2.0",
    "proxy-py~=2.4.0",
    "pydoc-markdown~=4.8.0",
    "pytest-asyncio~=0.26.0",
    "pytest-cov~=6.1.0",
    "pytest-only~=2.1.0",
    "pytest-xdist~=3.6.0",
    "pytest~=8.3.0",
    "ruff~=0.11.0",
    "setuptools~=79.0.0",                    # setuptools are used by pytest, but not explicitly required
    "sortedcontainers-stubs~=2.4.0",
    "types-beautifulsoup4~=4.12.0.20240229",
    "types-cachetools~=5.5.0.20240820",
    "types-colorama~=0.4.15.20240106",
    "types-psutil~=7.0.0.20250218",
    "types-python-dateutil~=2.9.0.20240316",
    "uvicorn[standard]~=0.34.0",
]

[tool.hatch.build.targets.wheel]
packages = ["src/crawlee"]

[tool.ruff]
line-length = 120
include = ["src/**/*.py", "tests/**/*.py", "docs/**/*.py", "website/**/*.py"]
extend-exclude = ["src/crawlee/project_template"]

[tool.ruff.lint]
select = ["ALL"]
ignore = [
    "ANN401",   # Dynamically typed expressions (typing.Any) are disallowed in {filename}
    "ASYNC109", # Async function definition with a `timeout` parameter
    "BLE001",   # Do not catch blind exception
    "C901",     # `{name}` is too complex
    "COM812",   # This rule may cause conflicts when used with the formatter
    "D100",     # Missing docstring in public module
    "D104",     # Missing docstring in public package
    "D107",     # Missing docstring in `__init__`
    "D203",     # One blank line required before class docstring
    "D213",     # Multi-line docstring summary should start at the second line
    "D413",     # Missing blank line after last section
    "EM",       # flake8-errmsg
    "G004",     # Logging statement uses f-string
    "ISC001",   # This rule may cause conflicts when used with the formatter
    "FIX",      # flake8-fixme
    "PLR0911",  # Too many return statements
    "PLR0913",  # Too many arguments in function definition
    "PLR0915",  # Too many statements
    "PTH",      # flake8-use-pathlib
    "PYI034",   # `__aenter__` methods in classes like `{name}` usually return `self` at runtime
    "PYI036",   # The second argument in `__aexit__` should be annotated with `object` or `BaseException | None`
    "S102",     # Use of `exec` detected
    "S105",     # Possible hardcoded password assigned to
    "S106",     # Possible hardcoded password assigned to argument: "{name}"
    "S301",     # `pickle` and modules that wrap it can be unsafe when used to deserialize untrusted data, possible security issue
    "S303",     # Use of insecure MD2, MD4, MD5, or SHA1 hash function
    "S311",     # Standard pseudo-random generators are not suitable for cryptographic purposes
    "TD002",    # Missing author in TODO; try: `# TODO(<author_name>): ...` or `# TODO @<author_name>: ...
    "TRY003",   # Avoid specifying long messages outside the exception class
]

[tool.ruff.format]
quote-style = "single"
indent-style = "space"

[tool.ruff.lint.per-file-ignores]
"**/__init__.py" = [
    "F401", # Unused imports
]
"**/{tests}/*" = [
    "D",       # Everything from the pydocstyle
    "INP001",  # File {filename} is part of an implicit namespace package, add an __init__.py
    "PLR2004", # Magic value used in comparison, consider replacing {value} with a constant variable
    "S101",    # Use of assert detected
    "SLF001",  # Private member accessed: `{name}`
    "T20",     # flake8-print
    "TRY301",  # Abstract `raise` to an inner function
]
"**/{docs,website}/**" = [
    "D",      # Everything from the pydocstyle
    "INP001", # File {filename} is part of an implicit namespace package, add an __init__.py
    "F841",   # Local variable {variable} is assigned to but never used
    "N999",   # Invalid module name
]
"**/docs/examples/code_examples/*crawler_with_error_snapshotter.py" = [
    "PLR2004", # Magic value used in comparison. Ignored for simplicity and readability of example code.
]
"**/docs/guides/code_examples/running_in_web_server/server.py" = [
    "TC002", # uv false positive. Import actually needed during runtime.
]

[tool.ruff.lint.flake8-quotes]
docstring-quotes = "double"
inline-quotes = "single"

[tool.ruff.lint.flake8-type-checking]
runtime-evaluated-base-classes = [
    "pydantic.BaseModel",
    "pydantic_settings.BaseSettings",
]

[tool.ruff.lint.flake8-builtins]
builtins-ignorelist = ["id"]

[tool.ruff.lint.isort]
known-first-party = ["crawlee"]

[tool.ruff.lint.pylint]
max-branches = 18

[tool.pytest.ini_options]
addopts = "-ra"
asyncio_default_fixture_loop_scope = "function"
asyncio_mode = "auto"
timeout = 1200

[tool.mypy]
python_version = "3.9"
plugins = ["pydantic.mypy"]
exclude = ["src/crawlee/project_template"]
files = ["src", "tests", "docs", "website"]
check_untyped_defs = true
disallow_incomplete_defs = true
disallow_untyped_calls = true
disallow_untyped_decorators = true
disallow_untyped_defs = true
no_implicit_optional = true
warn_redundant_casts = true
warn_return_any = true
warn_unreachable = true
warn_unused_ignores = true

[[tool.mypy.overrides]]
# Example codes are sometimes showing integration of crawlee with external tool, which is not dependency of crawlee.
module = [
    "apify",                        # Example code shows integration of apify and crawlee.
    "apify_fingerprint_datapoints", # Untyped and stubs not available
    "camoufox",                     # Example code shows integration of camoufox and crawlee.
    "fastapi",                      # Example code shows running in webserver.
    "starlette.*",                  # Example code shows running in webserver.
    "flask",                        # Example code shows deploy on Google Cloud.
    "functions_framework",          # Example code shows deploy on Google Cloud.
    "jaro",                         # Untyped and stubs not available
    "litestar",                     # Example code shows deploy on Google Cloud Run.
    "loguru",                       # Example code shows integration of loguru and crawlee for JSON logging.
    "protego",                      # Untyped and stubs not available
    "sklearn.linear_model",         # Untyped and stubs not available
    "sortedcollections",            # Untyped and stubs not available
    "cookiecutter.*",               # Untyped and stubs not available
    "inquirer.*",                   # Untyped and stubs not available
]
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = [
    "running_in_web_server.*"  # False positive when fastapi not available
]
disable_error_code = ["misc"]

[tool.basedpyright]
pythonVersion = "3.9"
typeCheckingMode = "standard"
include = ["src", "tests", "docs", "website"]

[tool.coverage.report]
exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:", "assert_never()"]

[tool.ipdb]
context = 7