pyproject.toml

[project]
name = "ol-data-platform"
version = "0.6.1"
description = "Code and configuration for powering the MIT Open Learning data platform"
authors = [{ name = "MIT Open Learning Engineering", email = "ol-data@mit.edu" }]
requires-python = "~=3.13,<3.15"
license = "BSD-3-Clause"
classifiers = [
    "Programming Language :: Python",
    "Programming Language :: Python :: 3",
]
dependencies = [
    "PyPika ~=0.51.0",
    "boto3>=1.42.42",
    "cyclopts~=4.10.0",
    "dbt-duckdb>=1.10.0",
    "dbt-trino>=1.10.0",
    "rich>=13.0",
    "sqlglot>=25.0",
    "trino[external-authentication-token-cache]>=0.336.0",
    "universal-pathlib ~=0.3.1",
]
repository = "https://github.com/mitodl/ol-data-platform"

[tool.uv.workspace]
members = ["packages/*", "src/ol_dbt_cli"]

[tool.uv]
package = false  # This is a workspace root, not a package

[dependency-groups]
dev = [
    "dagster-dg-cli>=1.12.10",
    "dagster-webserver>=1.12.10",
    "mypy",
    "ol-dbt-cli",
    "ol-orchestrate-lib",
    "pre-commit>=4.0.0,<5",
    "pytest>=9.0.1,<10",
    "pytest-cov>=6.0.0,<7",
    "ruff",
    "shandy-sqlfmt[jinjafmt]>=0.27.0",
    "sqlfluff[rs]>=4.0.0",
    "types-boto3>=1.42.65",
]

[tool.uv.sources]
ol-dbt-cli = {workspace = true}
ol-orchestrate-lib = { workspace = true }

# The ol_orchestrate package has been moved to packages/ol-orchestrate-lib/
# and is managed as a workspace member package

[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"

[tool.sqlfmt]
line_length = 120
exclude=["target/**/*", "dbt_packages/**/*"]

[tool.sqlfluff.core]
templater = "jinja"
dialect = "trino"
sql_file_exts = ".sql,.sql.j2,.dml,.ddl"
max_line_length = 120
ignore = "templating"
exclude_rules = [
  "layout.indent",
  "layout.cte_bracket",
  "layout.keyword_newline",
  "layout.cte_newline",
  "layout.select_targets",
  "layout.spacing",
  "convention.quoted_literals"
]
warn_unused_ignores = true

[tool.sqlfluff.rules]
capitalisation_policy = "lower"
extended_capitalisation_policy = "lower"

[tool.sqlfluff.rules.convention.terminator]
multiline_newline = true

[tool.sqlfluff.templater]
unwrap_wrapped_queries = true

[tool.sqlfluff.templater.jinja]
apply_dbt_builtins = true
load_macros_from_path = "src/ol_dbt/macros/"

[tool.sqlfluff.templater.dbt]
project_dir = "src/ol_dbt/"
profiles_dir = "src/ol_dbt/"
profile ="open_learning"
target = "qa"

[tool.mypy]
python_version = "3.12"
plugins = [
  "pydantic.mypy"
]
follow_imports = "normal"
warn_redundant_casts = true
warn_unused_ignores = true
disallow_any_generics = true
check_untyped_defs = true
no_implicit_reexport = true
ignore_missing_imports = true

[tool.pydantic-mypy]
init_forbid_extra = true
init_typed = true
warn_required_dynamic_aliases = true

[tool.ruff]
target-version = "py312"
line-length = 88
lint.select = [
    "A",  # flake8-builtins
    # "AIR",  # Airflow
    # "ANN",  # flake8-annotations
    "ARG",  # flake8-unused-arguments
    # "ASYNC",  # flake8-async
    "B",  # flake8-bugbear
    "BLE",  # flake8-blind-except
    "C4",  # flake8-comprehensions
    "C90",  # mccabe
    # "COM",  # flake8-commas
    "CPY",  # flake8-copyright
    "D",  # pydocstyle
    # "DJ",  # flake8-django
    "DTZ",  # flake8-datetimez
    "E",  # Pycodestyle Error
    "EM",  # flake8-errmsg
    "ERA",  # eradicate
    "EXE",  # flake8-executable
    "F",  # Pyflakes
    "FA",  # flake8-future-annotations
    "FBT",  # flake8-boolean-trap
    "FIX",  # flake8-fixme
    "FLY",  # flynt
    # "FURB",  # refurb
    "G",  # flake8-logging-format
    "I",  # isort
    "ICN",  # flake8-import-conventions
    "INP",  # flake8-no-pep420
    "INT",  # flake8-gettext
    "ISC",  # flake8-implicit-str-concat
    "N",  # pep8-naming
    # "NPY",  # NumPy-specific rules
    # "PD",  # pandas-vet
    "PERF",  # Perflint
    "PGH",  # pygrep-hooks
    "PIE",  # flake8-pie
    "PL",  # Pylint
    "PT",  # flake8-pytest-style
    "PTH",  # flake8-use-pathlib
    "PYI",  # flake8-pyi
    "Q",  # flake8-quotes
    "RET",  # flake8-return
    "RSE",  # flake8-raise
    "RUF",  # Ruff-specific rules
    "S",  # flake8-bandit
    "SIM",  # flake8-simplify
    "SLF",  # flake8-self
    "SLOT",  # flake8-slots
    "T10",  # flake8-debugger
    "T20",  # flake8-print
    # "TCH",  # flake8-type-checking
    "TD",  # flake8-todos
    "TID",  # flake8-tidy-imports
    "TRY",  # tryceratops
    "UP",  # pyupgrade
    "W",  # Pycodestyle Warning
    "YTT",  # flake8-2020
]
lint.ignore = [
    "B008",
    "B905",
    "D104",
    "D200",
    "D202",
    "D205",
    "D301",
    "D400",
    "N803",
    "N806",
    "N999",
    "PIE804",
    "RET505",
    "RET506",
    "RET507",
    "RET508",
    "SIM115",
    "UP007"
]
lint.typing-modules = ["colour.hints"]

[tool.ruff.lint.pydocstyle]
convention = "pep257"

[tool.ruff.lint.flake8-quotes]
inline-quotes = "double"

[tool.ruff.lint.flake8-builtins]
builtins-allowed-modules = ["secrets", "platform"]

[tool.ruff.lint.per-file-ignores]
"scripts/**" = ["T201", "ERA001", "B", "BLE001", "PLR", "E501", "G004", "FBT", "C90", "INP001"]
"**/tests/**" = ["S101", "PLR", "RUF059", "S106", "SLF001", "ERA001"]
"test_*.py" = ["S101"]

[tool.dg]
directory_type = "workspace"

[[tool.dg.workspace.projects]]
path = "dg_projects/lakehouse"

[[tool.dg.workspace.projects]]
path = "dg_projects/openedx"

[[tool.dg.workspace.projects]]
path = "dg_projects/edxorg"

[[tool.dg.workspace.projects]]
path = "dg_projects/legacy_openedx"

[[tool.dg.workspace.projects]]
path = "dg_projects/canvas"

[[tool.dg.workspace.projects]]
path = "dg_projects/data_platform"

[[tool.dg.workspace.projects]]
path = "dg_projects/learning_resources"

[[tool.dg.workspace.projects]]
path = "dg_projects/data_loading"