diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index fcfaa8eb..367010b5 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -15,7 +15,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.11 + python-version: 3.13 - name: Install Linters run: | pip install \ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bbfcbec3..63b7f464 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,6 @@ jobs: python-test: runs-on: ubuntu-latest steps: - - run: sudo apt-get install graphviz - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 @@ -29,10 +28,10 @@ jobs: python-test-install: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] + python-version: ["3.11", "3.12", "3.13", "3.14"] steps: - - run: sudo apt-get install graphviz - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 @@ -52,10 +51,10 @@ jobs: python-edge-test: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] + python-version: ["3.11", "3.12", "3.13", "3.14"] steps: - - run: sudo apt-get install graphviz - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 diff --git a/Dockerfile b/Dockerfile index f11db819..158ed5e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,17 +4,14 @@ # pattern is able to reduce each image to ~250MB but takes considerable # time to build and is considerably more complex for scipy and pandas. -FROM node:20.19-bullseye AS frontend +FROM node:24.10-trixie AS frontend WORKDIR /app COPY ./nereid/nereid/static/frontend . RUN npm install . && npm run build CMD ["bash", "-c", "while true; do sleep 1; done"] -FROM python:3.11.12-bullseye AS nereid_install -RUN apt-get update -y \ - && apt-get install -y --no-install-recommends graphviz \ - && rm -rf /var/lib/apt/lists/* +FROM python:3.13.8-trixie AS nereid_install WORKDIR /nereid CMD ["bash", "-c", "while true; do sleep 1; done"] @@ -24,9 +21,9 @@ COPY nereid/redis.conf /redis.conf CMD ["redis-server", "/redis.conf"] -FROM python:3.11.12-alpine3.21 AS flower +FROM python:3.13.8-alpine3.22 AS flower RUN apk add --no-cache ca-certificates tzdata && update-ca-certificates -RUN pip install --no-cache-dir redis==4.6.0 flower==1.0.0 celery==5.3.4 +RUN pip install --no-cache-dir redis==5.2.1 flower==2.0.1 celery==5.5.3 ENV PYTHONUNBUFFERED=1 PYTHONHASHSEED=random PYTHONDONTWRITEBYTECODE=1 ENV FLOWER_DATA_DIR=/data ENV PYTHONPATH=${FLOWER_DATA_DIR} @@ -45,7 +42,7 @@ EXPOSE 5555 CMD ["celery", "flower"] -FROM python:3.11.12-bullseye AS core-env +FROM python:3.13.8-trixie AS core-env RUN apt-get update && apt-get install -y build-essential curl ADD https://astral.sh/uv/0.6.14/install.sh /install.sh RUN sh /install.sh && rm /install.sh @@ -70,10 +67,7 @@ RUN uv pip install --no-cache \ -r /requirements_dev.txt -FROM python:3.11.12-slim-bullseye AS core-runtime -RUN apt-get update -y \ - && apt-get install -y --no-install-recommends graphviz \ - && rm -rf /var/lib/apt/lists/* +FROM python:3.13.8-slim-trixie AS core-runtime WORKDIR /nereid ENV PYTHONPATH=/nereid ENV PATH=/opt/venv/bin:$PATH @@ -111,10 +105,7 @@ COPY --chmod=755 nereid/scripts / CMD ["bash", "-c", "while true; do sleep 1; done"] -FROM python:3.12-bullseye AS nereid-edge -RUN apt-get update -y \ - && apt-get install -y graphviz build-essential curl \ - && rm -rf /var/lib/apt/lists/* +FROM python:3.14-trixie AS nereid-edge ADD https://astral.sh/uv/0.6.14/install.sh /install.sh RUN sh /install.sh && rm /install.sh ENV PATH=/opt/venv/bin:/root/.local/bin/:$PATH diff --git a/nereid/nereid/api/api_v1/endpoints_async/land_surface_loading.py b/nereid/nereid/api/api_v1/endpoints_async/land_surface_loading.py index dfbec3d3..c2f2e304 100644 --- a/nereid/nereid/api/api_v1/endpoints_async/land_surface_loading.py +++ b/nereid/nereid/api/api_v1/endpoints_async/land_surface_loading.py @@ -28,7 +28,7 @@ async def calculate_loading( ) -> dict[str, Any]: land_surfaces_req = land_surfaces.model_dump(by_alias=True) - task = bg.land_surface_loading.s( + task = bg.land_surface_loading.s( # type: ignore land_surfaces=land_surfaces_req, details=details, context=context ) @@ -45,7 +45,7 @@ async def get_land_surface_loading_result( request: Request, task_id: str, ) -> dict[str, Any]: - task = bg.land_surface_loading.AsyncResult(task_id, app=router) + task = bg.land_surface_loading.AsyncResult(task_id, app=router) # type: ignore return await standard_json_response( request, task, "get_land_surface_loading_result" ) diff --git a/nereid/nereid/api/api_v1/endpoints_async/network.py b/nereid/nereid/api/api_v1/endpoints_async/network.py index 09c1665d..bca4bdc7 100644 --- a/nereid/nereid/api/api_v1/endpoints_async/network.py +++ b/nereid/nereid/api/api_v1/endpoints_async/network.py @@ -29,7 +29,7 @@ async def validate_network( openapi_examples=network_models.GraphExamples, # type: ignore[arg-type] ), ) -> dict[str, Any]: - task = bg.validate_network.s(graph=graph.model_dump(by_alias=True)) + task = bg.validate_network.s(graph=graph.model_dump(by_alias=True)) # type: ignore return await run_task(request, task, "get_validate_network_result") @@ -40,7 +40,7 @@ async def validate_network( response_class=ORJSONResponse, ) async def get_validate_network_result(request: Request, task_id: str) -> dict[str, Any]: - task = bg.validate_network.AsyncResult(task_id, app=router) + task = bg.validate_network.AsyncResult(task_id, app=router) # type: ignore return await standard_json_response(request, task, "get_validate_network_result") @@ -54,7 +54,7 @@ async def subgraph_network( request: Request, subgraph_req: network_models.SubgraphRequest = Body(...), ) -> dict[str, Any]: - task = bg.network_subgraphs.s(**subgraph_req.model_dump(by_alias=True)) + task = bg.network_subgraphs.s(**subgraph_req.model_dump(by_alias=True)) # type: ignore return await run_task(request, task, "get_subgraph_network_result") @@ -66,7 +66,7 @@ async def subgraph_network( response_class=ORJSONResponse, ) async def get_subgraph_network_result(request: Request, task_id: str) -> dict[str, Any]: - task = bg.network_subgraphs.AsyncResult(task_id, app=router) + task = bg.network_subgraphs.AsyncResult(task_id, app=router) # type: ignore return await standard_json_response(request, task, "get_subgraph_network_result") @@ -81,26 +81,26 @@ async def get_subgraph_network_as_img( task_id: str, media_type: str = Query("svg"), npi: float = Query(4.0), + timeout: float | None = Query(None, le=10.0, gt=0.0), ) -> dict[str, Any] | Any: if media_type != "svg": detail = f"media_type not supported: '{media_type}'." raise HTTPException(status_code=400, detail=detail) - task = bg.network_subgraphs.AsyncResult(task_id, app=router) + task = bg.network_subgraphs.AsyncResult(task_id, app=router) # type: ignore response = {"task_id": task.task_id, "status": task.status} if task.successful(): result = task.result response["data"] = task.result render_task_id = task.task_id + f"-{media_type}-{npi}" - - render_task = bg.render_subgraph_svg.AsyncResult(render_task_id, app=router) + render_task = bg.render_subgraph_svg.AsyncResult(render_task_id, app=router) # type: ignore if not render_task.ready() and render_task.status.lower() != "started": - render_task = bg.render_subgraph_svg.apply_async( + render_task = bg.render_subgraph_svg.apply_async( # type: ignore args=(result, npi), task_id=render_task_id ) _ = await wait_a_sec_and_see_if_we_can_return_some_data( - render_task, timeout=10 + render_task, timeout=timeout or 5.0 ) if render_task.successful(): @@ -125,7 +125,7 @@ async def network_solution_sequence( ), min_branch_size: int = Query(4), ) -> dict[str, Any]: - task = bg.solution_sequence.s( + task = bg.solution_sequence.s( # type: ignore graph=graph.model_dump(by_alias=True), min_branch_size=min_branch_size ) @@ -141,7 +141,7 @@ async def network_solution_sequence( async def get_network_solution_sequence( request: Request, task_id: str ) -> dict[str, Any]: - task = bg.solution_sequence.AsyncResult(task_id, app=router) + task = bg.solution_sequence.AsyncResult(task_id, app=router) # type: ignore return await standard_json_response(request, task, "get_network_solution_sequence") @@ -156,12 +156,13 @@ async def get_network_solution_sequence_as_img( task_id: str, media_type: str = Query("svg"), npi: float = Query(4.0), + timeout: float | None = Query(None, le=10.0, gt=0.0), ) -> dict[str, Any] | Any: if media_type != "svg": detail = f"media_type not supported: '{media_type}'." raise HTTPException(status_code=400, detail=detail) - task = bg.solution_sequence.AsyncResult(task_id, app=router) + task = bg.solution_sequence.AsyncResult(task_id, app=router) # type: ignore response = {"task_id": task.task_id, "status": task.status} if task.successful(): @@ -169,15 +170,15 @@ async def get_network_solution_sequence_as_img( response["data"] = task.result render_task_id = task.task_id + f"-{media_type}-{npi}" - render_task = bg.render_solution_sequence_svg.AsyncResult( + render_task = bg.render_solution_sequence_svg.AsyncResult( # type: ignore render_task_id, app=router ) if not render_task.ready() and render_task.status.lower() != "started": - render_task = bg.render_solution_sequence_svg.apply_async( + render_task = bg.render_solution_sequence_svg.apply_async( # type: ignore args=(result, npi), task_id=render_task_id ) _ = await wait_a_sec_and_see_if_we_can_return_some_data( - render_task, timeout=10 + render_task, timeout=timeout or 5.0 ) if render_task.successful(): diff --git a/nereid/nereid/api/api_v1/endpoints_async/tasks.py b/nereid/nereid/api/api_v1/endpoints_async/tasks.py index 97c2b90e..b708a2ed 100644 --- a/nereid/nereid/api/api_v1/endpoints_async/tasks.py +++ b/nereid/nereid/api/api_v1/endpoints_async/tasks.py @@ -12,13 +12,13 @@ @router.get("/ping", response_model=JSONAPIResponse) async def get_ping(request: Request) -> dict[str, Any]: # pragma: no cover - task = bg.background_ping.apply_async() + task = bg.background_ping.apply_async() # type: ignore return await standard_json_response(request, task) @router.get("/sleep", response_model=JSONAPIResponse) async def get_sleep(request: Request, s: int = 1) -> dict[str, Any]: # pragma: no cover - task = bg.background_sleep.s(seconds=s).apply_async() + task = bg.background_sleep.s(seconds=s).apply_async() # type: ignore return await standard_json_response(request, task) diff --git a/nereid/nereid/api/api_v1/endpoints_async/treatment_facilities.py b/nereid/nereid/api/api_v1/endpoints_async/treatment_facilities.py index bdb015f3..fce942dc 100644 --- a/nereid/nereid/api/api_v1/endpoints_async/treatment_facilities.py +++ b/nereid/nereid/api/api_v1/endpoints_async/treatment_facilities.py @@ -43,7 +43,7 @@ async def initialize_treatment_facility_parameters( ) -> dict[str, Any]: treatment_facilities, context = tmnt_facility_req - task = bg.initialize_treatment_facilities.s( + task = bg.initialize_treatment_facilities.s( # type: ignore treatment_facilities=treatment_facilities.model_dump(), pre_validated=True, context=context, @@ -60,7 +60,7 @@ async def initialize_treatment_facility_parameters( async def get_treatment_facility_parameters( request: Request, task_id: str ) -> dict[str, Any]: - task = bg.initialize_treatment_facilities.AsyncResult(task_id, app=router) + task = bg.initialize_treatment_facilities.AsyncResult(task_id, app=router) # type: ignore return await standard_json_response( request, task, "get_treatment_facility_parameters" ) diff --git a/nereid/nereid/api/api_v1/endpoints_async/treatment_sites.py b/nereid/nereid/api/api_v1/endpoints_async/treatment_sites.py index 529fa14b..fc705433 100644 --- a/nereid/nereid/api/api_v1/endpoints_async/treatment_sites.py +++ b/nereid/nereid/api/api_v1/endpoints_async/treatment_sites.py @@ -25,7 +25,7 @@ async def initialize_treatment_site( treatment_sites: TreatmentSites = Body(...), context: dict = Depends(get_valid_context), ) -> dict[str, Any]: - task = bg.initialize_treatment_sites.s( + task = bg.initialize_treatment_sites.s( # type: ignore treatment_sites.model_dump(), context=context ) @@ -41,5 +41,5 @@ async def initialize_treatment_site( async def get_treatment_site_parameters( request: Request, task_id: str ) -> dict[str, Any]: - task = bg.initialize_treatment_sites.AsyncResult(task_id, app=router) + task = bg.initialize_treatment_sites.AsyncResult(task_id, app=router) # type: ignore return await standard_json_response(request, task, "get_treatment_site_parameters") diff --git a/nereid/nereid/api/api_v1/endpoints_async/watershed.py b/nereid/nereid/api/api_v1/endpoints_async/watershed.py index ceef5000..429e4046 100644 --- a/nereid/nereid/api/api_v1/endpoints_async/watershed.py +++ b/nereid/nereid/api/api_v1/endpoints_async/watershed.py @@ -44,7 +44,7 @@ async def post_solve_watershed( ), ) -> dict[str, Any]: watershed, context = watershed_pkg - task = bg.solve_watershed.s( + task = bg.solve_watershed.s( # type: ignore watershed=watershed, treatment_pre_validated=True, context=context ) return await run_task(request, task, "get_watershed_result") @@ -57,5 +57,5 @@ async def post_solve_watershed( response_class=ORJSONResponse, ) async def get_watershed_result(request: Request, task_id: str) -> dict[str, Any]: - task = bg.solve_watershed.AsyncResult(task_id, app=router) + task = bg.solve_watershed.AsyncResult(task_id, app=router) # type: ignore return await standard_json_response(request, task, "get_watershed_result") diff --git a/nereid/nereid/api/api_v1/endpoints_sync/network.py b/nereid/nereid/api/api_v1/endpoints_sync/network.py index 9a4417d5..cf80fa7d 100644 --- a/nereid/nereid/api/api_v1/endpoints_sync/network.py +++ b/nereid/nereid/api/api_v1/endpoints_sync/network.py @@ -1,10 +1,13 @@ from typing import Any +import networkx as nx from fastapi import APIRouter, Body, Query -from fastapi.responses import ORJSONResponse +from fastapi.responses import ORJSONResponse, StreamingResponse from nereid.models import network_models from nereid.src import tasks +from nereid.src.network.algorithms import parallel_sequential_subgraph_nodes +from nereid.src.network.utils import graph_factory, thin_graph_dict router = APIRouter() @@ -18,7 +21,7 @@ async def validate_network( graph: network_models.Graph = Body( ..., - openapi_examples=network_models.GraphExamples, # type: ignore[arg-type] + openapi_examples=network_models.GraphExamples, ), ) -> dict[str, Any]: g: dict[str, Any] = graph.model_dump(by_alias=True) @@ -40,6 +43,32 @@ async def subgraph_network( return {"data": data} +@router.post( + "/network/subgraph/img", + tags=["network", "subgraph"], +) +async def subgraph_network_as_img( + subgraph_req: network_models.SubgraphRequest = Body(...), + npi: float = Query(4.0), +) -> StreamingResponse: + kwargs = subgraph_req.model_dump(by_alias=True) + graph_dict = kwargs["graph"] + nodes = kwargs["nodes"] + + res = tasks.network_subgraphs(graph_dict, nodes) + g = graph_factory(res["graph"]) + + fig = tasks.render_subgraphs( + g, + request_nodes=res["requested_nodes"], + subgraph_nodes=res["subgraph_nodes"], + npi=npi, + ) + + svg = tasks.fig_to_image(fig, format="jpg") + return StreamingResponse(svg, media_type="image/jpg") + + @router.post( "/network/solution_sequence", tags=["network", "sequence"], @@ -49,10 +78,32 @@ async def subgraph_network( async def network_solution_sequence( graph: network_models.ValidGraph = Body( ..., - openapi_examples=network_models.GraphExamples, # type: ignore[arg-type] + openapi_examples=network_models.GraphExamples, ), min_branch_size: int = Query(4), ) -> dict[str, Any]: g = graph.model_dump(by_alias=True) data = tasks.solution_sequence(graph=g, min_branch_size=min_branch_size) return {"data": data} + + +@router.post("/network/solution_sequence/img", tags=["network", "sequence"]) +async def network_solution_sequence_as_img( + graph: network_models.ValidGraph = Body( + ..., + openapi_examples=network_models.GraphExamples, + ), + min_branch_size: int = Query(4), + npi: float = Query(4.0), +) -> StreamingResponse: + _g = thin_graph_dict(graph.model_dump(by_alias=True)) + g = nx.DiGraph(graph_factory(_g)) + + solution_sequence = parallel_sequential_subgraph_nodes(g, min_branch_size) + + fig = tasks.render_solution_sequence( + g, solution_sequence=solution_sequence, npi=npi + ) + + svg = tasks.fig_to_image(fig, format="jpg") + return StreamingResponse(svg, media_type="image/jpg") diff --git a/nereid/nereid/bg_worker.py b/nereid/nereid/bg_worker.py index d767ea8c..b65c4aa9 100644 --- a/nereid/nereid/bg_worker.py +++ b/nereid/nereid/bg_worker.py @@ -15,6 +15,7 @@ accept_content=["json"], # Ignore other content result_serializer="json", broker_connection_retry_on_startup=True, + worker_cancel_long_running_tasks_on_connection_loss=True, ) inspector = celery_app.control.inspect() diff --git a/nereid/nereid/core/io.py b/nereid/nereid/core/io.py index 94c53f2b..b7db01c1 100644 --- a/nereid/nereid/core/io.py +++ b/nereid/nereid/core/io.py @@ -215,7 +215,7 @@ def parse_joins( ) if "warnings" not in df: df["warnings"] = "" - df.loc[df["_merge"] != "both", "warnings"] += ( + df.loc[df["_merge"] != "both", "warnings"] += ( # type: ignore f"Warning: unable join '{j['left_on']}' from {tablename} to '{j['right_on']}' " f"for reference data in {config_section}:{config_object}.\n" ) diff --git a/nereid/nereid/factory.py b/nereid/nereid/factory.py index a92769b1..a54d3021 100644 --- a/nereid/nereid/factory.py +++ b/nereid/nereid/factory.py @@ -47,20 +47,19 @@ def create_app( static_path = nereid_path / "static" app.mount("/static", StaticFiles(directory=static_path), name="static") - if _settings.ASYNC_MODE == "replace": # pragma: no cover + if _settings.ASYNC_MODE == "replace": from nereid.api import async_router app.include_router(async_router, tags=["async"]) - else: - app.include_router(sync_router) - if _settings.ASYNC_MODE == "add": # pragma: no cover - from nereid.api import async_router - - app.include_router( - async_router, - prefix=_settings.ASYNC_ROUTE_PREFIX, - tags=["async"], - ) + elif _settings.ASYNC_MODE == "add": # pragma: no branch + from nereid.api import async_router + + app.include_router( + async_router, + prefix=_settings.ASYNC_ROUTE_PREFIX, + tags=["async"], + ) + app.include_router(sync_router) @app.get("/docs", include_in_schema=False) async def custom_swagger_ui_html(req: Request) -> HTMLResponse: diff --git a/nereid/nereid/models/treatment_facility_models.py b/nereid/nereid/models/treatment_facility_models.py index 9e8ae3b8..09e7074e 100644 --- a/nereid/nereid/models/treatment_facility_models.py +++ b/nereid/nereid/models/treatment_facility_models.py @@ -212,9 +212,7 @@ class TmntFacilityWithRetentionOverride(TmntFacility): class FlowAndRetFacility(FlowFacility, FacilityBase): area_sqft: FLOAT_NON_ZERO - - # TODO: if hsg = lined then users may enter zero for the depth. - depth_ft: FLOAT_NON_ZERO + depth_ft: FLOAT_GE_ZERO hsg: str _constructor: str = "flow_and_retention_facility_constructor" diff --git a/nereid/nereid/src/network/algorithms.py b/nereid/nereid/src/network/algorithms.py index 074de17f..c307b348 100644 --- a/nereid/nereid/src/network/algorithms.py +++ b/nereid/nereid/src/network/algorithms.py @@ -8,9 +8,15 @@ def find_cycle(g: nx.Graph, **kwargs: dict) -> list: if no cycle is found. """ try: - return list(nx.find_cycle(g, **kwargs)) + return list( + nx.find_cycle( + g, + source=kwargs.pop("source", None), + orientation=kwargs.pop("orientation", None), # type: ignore + ) + ) - except nx.exception.NetworkXNoCycle: + except nx.NetworkXNoCycle: return [] @@ -35,7 +41,7 @@ def get_subset(g: nx.DiGraph, nodes: Hashable | Iterable[Hashable]) -> set: def get_all_predecessors( g: nx.DiGraph, node: Hashable, subset: set | None = None ) -> set: - """This algorithm is a good deal faster than the nx.ancestors variant, + """This algorithm is a good deal faster (~7x-20x) than the nx.ancestors variant, **but** it only works on directed acyclic graphs (DAGs). """ if subset is None: @@ -77,11 +83,12 @@ def find_leafy_branch_larger_than_size(g: nx.DiGraph, size: int = 1) -> nx.DiGra us = get_all_predecessors(g, node) us.add(node) if len(us) >= size: - return g.subgraph(us) + return nx.DiGraph(g.subgraph(us)) return nx.DiGraph() # pragma: no cover -def sequential_subgraph_nodes(g: nx.DiGraph, size: int) -> list[list[Hashable]]: +def sequential_subgraph_nodes(g: nx.Graph, size: int) -> list[list[Hashable]]: + g = nx.DiGraph(g) # make a copy because we'll modify the structure if not nx.is_weakly_connected(g): raise nx.NetworkXUnfeasible( "sequential solutions are not possible for disconnected graphs." @@ -90,18 +97,16 @@ def sequential_subgraph_nodes(g: nx.DiGraph, size: int) -> list[list[Hashable]]: if size < 2: raise nx.NetworkXUnfeasible("the minimum directed subgraph length is 2 nodes.") - g = nx.DiGraph(g.edges()) # make a copy because we'll modify the structure - graphs = [] - while len(g.nodes()) > 1: + while len(g.nodes) > 1: sg = find_leafy_branch_larger_than_size(g, size) - graphs.append(list(sg.nodes())) + graphs.append(list(sg.nodes)) # trim the upstream nodes out of the graph, except the upstream root us_nodes = {n for n, deg in sg.out_degree if deg > 0} - g = g.subgraph(g.nodes() - us_nodes) + g = nx.DiGraph(g.subgraph(g.nodes - us_nodes)) # rinse and repeat until there's one or fewer nodes left in the graph diff --git a/nereid/nereid/src/network/render.py b/nereid/nereid/src/network/render.py index ac49b6dc..fcc0bd93 100644 --- a/nereid/nereid/src/network/render.py +++ b/nereid/nereid/src/network/render.py @@ -1,11 +1,8 @@ -import warnings -from functools import lru_cache from io import BytesIO from itertools import cycle from typing import IO, TYPE_CHECKING, Any, cast import networkx as nx -import orjson as json if TYPE_CHECKING: from matplotlib.axes import Axes @@ -15,28 +12,6 @@ Figure = Any -def pydot_layout(*args, **kwargs) -> dict: - with warnings.catch_warnings(): - warnings.filterwarnings(action="ignore", message="nx.nx_pydot") - v = cast(dict, nx.nx_pydot.pydot_layout(*args, **kwargs)) - return v - - -@lru_cache(maxsize=100) -def _cached_layout(edge_json: str, prog: str) -> dict[str | int, tuple[float, float]]: - g = nx.from_edgelist(json.loads(edge_json), create_using=nx.MultiDiGraph) - layout: dict[str | int, tuple[float, float]] = pydot_layout(g, prog=prog) - return layout - - -def cached_layout( - g: nx.Graph, prog: str = "dot" -) -> dict[str | int, tuple[float, float]]: - edges = sorted(g.edges(), key=lambda x: str(x)) - edge_json = json.dumps(list(edges)) - return _cached_layout(edge_json, prog=prog) - - def get_figure_width_height_from_graph_layout( layout_dict: dict[str | int, tuple[float, float]], npi: float | None = None, @@ -47,7 +22,7 @@ def get_figure_width_height_from_graph_layout( Parameters ---------- layout_dict : dict e.g., {'node_id': (x, y), ...} - layout mapping created by nx.nx_pydot.pydot_layout + layout mapping npi : float, optional (default=3) nodes per inch of the final figure min_width : float, optional (default=1) in inches @@ -61,7 +36,10 @@ def get_figure_width_height_from_graph_layout( xmax = max([v[0] for v in layout_dict.values()]) ymax = max([v[1] for v in layout_dict.values()]) - aspect = ymax / xmax + if xmax > 0: + aspect = ymax / xmax + else: # pragma: no cover + aspect = 1 width = min_width + (ndots / npi) height = min_height + (ndots / npi) * aspect @@ -80,12 +58,13 @@ def render_subgraphs( fig_kwargs: dict | None = None, ) -> Figure: import matplotlib.pyplot as plt + from fast_sugiyama import from_edges if fig_kwargs is None: # pragma: no branch fig_kwargs = {} if layout is None: # pragma: no branch - layout = cached_layout(g, prog="dot") + layout = from_edges(g.edges()).dot_layout().to_dict() if "figsize" not in fig_kwargs: # pragma: no branch width, height = get_figure_width_height_from_graph_layout(layout, npi=npi) @@ -158,10 +137,11 @@ def render_solution_sequence( fig_kwargs: dict | None = None, ) -> Figure: import matplotlib.pyplot as plt + from fast_sugiyama import from_edges from matplotlib import colormaps if layout is None: # pragma: no branch - layout = cached_layout(G, prog="dot") + layout = from_edges(G.edges()).dot_layout().to_dict() if cmap_str is None: # pragma: no branch cmap_str = "Blues_r" if fig_kwargs is None: # pragma: no branch diff --git a/nereid/nereid/src/network/tasks.py b/nereid/nereid/src/network/tasks.py index 59aad7c1..ea9abccb 100644 --- a/nereid/nereid/src/network/tasks.py +++ b/nereid/nereid/src/network/tasks.py @@ -35,15 +35,11 @@ def validate_network(graph: dict) -> dict[str, bool | list]: result: dict[str, bool | list] = {"isvalid": isvalid} - if isvalid: - return result - - else: + if not isvalid: _keys = ["node_cycles", "edge_cycles", "multiple_out_edges", "duplicate_edges"] - for key, value in zip(_keys, validate.validate_network(g), strict=True): - result[key] = value # noqa [PERF403] + result.update(**dict(zip(_keys, validate.validate_network(g), strict=True))) - return result + return result def network_subgraphs( @@ -55,7 +51,7 @@ def network_subgraphs( g = nx.DiGraph(graph_factory(_graph)) subset = get_subset(g, node_ids) - sub_g = g.subgraph(subset) + sub_g = nx.DiGraph(g.subgraph(subset)) subgraph_nodes = [ {"nodes": [{"id": n} for n in nodes]} @@ -87,7 +83,7 @@ def render_subgraph_svg(task_result: dict, npi: float | None = None) -> bytes: def solution_sequence( graph: dict[str, Any], min_branch_size: int -) -> dict[str, dict[str, list[dict[str, list[dict[str, str | dict]]]]]]: +) -> dict[str, dict[str, list[dict[str, list[dict[str, dict]]]]]]: _graph = thin_graph_dict(graph) # strip unneeded metadata g = nx.DiGraph(graph_factory(_graph)) diff --git a/nereid/nereid/src/network/validate.py b/nereid/nereid/src/network/validate.py index 1a631e5f..766d70b2 100644 --- a/nereid/nereid/src/network/validate.py +++ b/nereid/nereid/src/network/validate.py @@ -36,20 +36,20 @@ def validate_network( def is_valid(G: GraphType) -> bool: - try: - # catch cycles - deque(nx.topological_sort(G), maxlen=0) - except nx.exception.NetworkXUnfeasible: + # catch multigraph + if len(G.edges()) != len(set(G.edges())): return False - try: - out_degs = nx.DiGraph(G).out_degree() - # catch multiple out connections - assert all((v <= 1 for _, v in out_degs)) # type: ignore + G = nx.DiGraph(G) - # catch - assert len(G.edges()) == len(set(G.edges())) - except Exception: + # catch multiple out connections + if not all((v <= 1 for _, v in G.out_degree())): # type: ignore + return False + + try: + # catch cycles + deque(nx.topological_sort(G), maxlen=0) + except nx.NetworkXUnfeasible: return False return True diff --git a/nereid/nereid/src/nomograph/interpolators.py b/nereid/nereid/src/nomograph/interpolators.py index 540966fa..cffb860b 100644 --- a/nereid/nereid/src/nomograph/interpolators.py +++ b/nereid/nereid/src/nomograph/interpolators.py @@ -303,14 +303,14 @@ def __call__( def _baseplot(self, ax: Axes | None = None, **kwargs: dict[str, Any]) -> Axes: if ax is None: # pragma: no branch - _, ax = cast(tuple[Any, Axes], plt.subplots()) + _, ax = cast(tuple[Any, Axes], plt.subplots()) # type: ignore fits = self(x=self.x_data, t=self.t_data) ax.scatter( self.x_data, fits, - marker="o", # type: ignore + marker="o", s=50, facecolor="none", edgecolor="k", @@ -332,12 +332,11 @@ def _baseplot(self, ax: Axes | None = None, **kwargs: dict[str, Any]) -> Axes: def _basesurface(self, ax: Axes | None = None, **kwargs: dict[str, Any]) -> Axes: if ax is None: # pragma: no branch - _, ax = cast(tuple[Any, Axes], plt.subplots()) + _, ax = cast(tuple[Any, Axes], plt.subplots()) # type: ignore ymax = numpy.nanmax(self.t_data) - grid_x, grid_y = numpy.mgrid[ - 0 : int(numpy.nanmax(self.x_data)) : 700j, 0 : int(ymax) : 700j # type: ignore[misc] - ] + xmax = numpy.nanmax(self.x_data) + grid_x, grid_y = numpy.mgrid[0:xmax:700j, 0:ymax:700j] # type: ignore[misc] grid_z = griddata( numpy.column_stack((self.x_data, self.t_data)), self.y_data, @@ -346,7 +345,7 @@ def _basesurface(self, ax: Axes | None = None, **kwargs: dict[str, Any]) -> Axes ) ax.imshow( grid_z.T, - extent=(0, numpy.nanmax(self.x_data), 0, ymax), + extent=(0, xmax, 0, ymax), origin="lower", aspect="auto", ) diff --git a/nereid/nereid/src/tasks.py b/nereid/nereid/src/tasks.py index 6770a3f5..5f5914f1 100644 --- a/nereid/nereid/src/tasks.py +++ b/nereid/nereid/src/tasks.py @@ -1,9 +1,14 @@ from nereid.src.land_surface.tasks import land_surface_loading as land_surface_loading +from nereid.src.network.tasks import fig_to_image as fig_to_image from nereid.src.network.tasks import network_subgraphs as network_subgraphs +from nereid.src.network.tasks import ( + render_solution_sequence as render_solution_sequence, +) from nereid.src.network.tasks import ( render_solution_sequence_svg as render_solution_sequence_svg, ) from nereid.src.network.tasks import render_subgraph_svg as render_subgraph_svg +from nereid.src.network.tasks import render_subgraphs as render_subgraphs from nereid.src.network.tasks import solution_sequence as solution_sequence from nereid.src.network.tasks import validate_network as validate_network from nereid.src.treatment_facility.tasks import ( diff --git a/nereid/nereid/src/watershed/solve_watershed.py b/nereid/nereid/src/watershed/solve_watershed.py index aeb17b07..641480ad 100644 --- a/nereid/nereid/src/watershed/solve_watershed.py +++ b/nereid/nereid/src/watershed/solve_watershed.py @@ -1,3 +1,4 @@ +from datetime import datetime, timezone from typing import Any, Callable import networkx as nx @@ -195,7 +196,7 @@ def solve_node( """ data = g.nodes[node] data["_current_node"] = node - data["_visited"] = True + data["_visited_at"] = datetime.now(timezone.utc).isoformat() data["node_errors"] = [] data["node_warnings"] = [] data["_is_leaf"] = False diff --git a/nereid/nereid/src/watershed/treatment_facility_capture.py b/nereid/nereid/src/watershed/treatment_facility_capture.py index f08a7349..a9640887 100644 --- a/nereid/nereid/src/watershed/treatment_facility_capture.py +++ b/nereid/nereid/src/watershed/treatment_facility_capture.py @@ -139,9 +139,12 @@ def compute_volume_based_standalone_facility( ret_ddt_hr = data.get("retention_ddt_hr", 0.0) trt_ddt_hr = data.get("treatment_ddt_hr", 0.0) + ret_depth_ft = data.get("retention_depth_ft", 0.0) + trt_depth_ft = data.get("treatment_depth_ft", 0.0) + design_volume = data["design_volume_cuft_cumul"] - if ret_vol_cuft > 0 and trt_vol_cuft > 0: + if ret_depth_ft > 1 / 12 and trt_depth_ft > 1 / 12: q_ret = safe_divide(ret_vol_cuft, (ret_ddt_hr * 3600)) q_trt = safe_divide(trt_vol_cuft, (trt_ddt_hr * 3600)) q_tot = q_ret + q_trt diff --git a/nereid/nereid/startup.py b/nereid/nereid/startup.py index 8732de25..d9e1e296 100644 --- a/nereid/nereid/startup.py +++ b/nereid/nereid/startup.py @@ -43,7 +43,7 @@ def get_worker(): import nereid.bg_worker as bg try: - bg.background_ping.apply_async().get(timeout=0.2) + bg.background_ping.apply_async().get(timeout=0.2) # type: ignore except Exception as e: logger.error(e) raise e diff --git a/nereid/nereid/tests/test_api/conftest.py b/nereid/nereid/tests/test_api/conftest.py index 12cb55c7..c4ac1a72 100644 --- a/nereid/nereid/tests/test_api/conftest.py +++ b/nereid/nereid/tests/test_api/conftest.py @@ -13,7 +13,7 @@ @pytest.fixture(scope="module") def client(async_mode): - mode = "none" + mode = "add" if async_mode: mode = "replace" app = create_app(settings_override={"ASYNC_MODE": mode}) @@ -102,8 +102,11 @@ def solution_sequence_response(client): responses[(bs, ngraph, minmax)] = response result_route = response.json().get("result_route") - if all([minmax == (10, 11), ngraph == 3, bs == 6, result_route]): - client.get(result_route + "/img?media_type=svg") + if all([minmax in [(3, 4), (10, 11)], ngraph in [1, 3], bs == 6]): + if result_route: + client.get(result_route + "/img?media_type=svg") + else: + client.post(route + f"/img?min_branch_size={bs}", json=payload) time.sleep(0.5) yield responses diff --git a/nereid/nereid/tests/test_api/test_network.py b/nereid/nereid/tests/test_api/test_network.py index afaef6c0..16187b7a 100644 --- a/nereid/nereid/tests/test_api/test_network.py +++ b/nereid/nereid/tests/test_api/test_network.py @@ -163,22 +163,34 @@ def test_get_render_subgraph_svg_fast( assert svg_response.status_code == 200 -def test_get_render_subgraph_svg_slow(client): - route = "api/v1/network/subgraph" - - slow_graph = clean_graph_dict(nx.gnr_graph(200, p=0.05, seed=42)) +@pytest.mark.parametrize("timeout", [0.000001, 5.0]) +def test_get_render_subgraph_svg_slow(client, timeout): + slow_graph = clean_graph_dict(nx.gn_graph(300, kernel=lambda x: 1, seed=42)) nodes = [{"id": "3"}, {"id": "29"}, {"id": "18"}] payload = {"graph": slow_graph, "nodes": nodes} + # test async + route = "api/v1/network/subgraph" response = client.post(route, json=payload) result_route = response.json().get("result_route") + q = f"?timeout={timeout:0.9f}" if timeout else "" if result_route: + svg_response = client.get(result_route + "/img" + q) + if "html" not in svg_response.headers["content-type"]: # pragma: no branch + srjson = response.json() + assert srjson["status"].lower() != "failure" + assert srjson["task_id"] is not None + svg_response = poll_testclient_url(client, result_route + "/img" + q) + + assert "DOCTYPE svg PUBLIC" in svg_response.content.decode() svg_response = client.get(result_route + "/img") assert svg_response.status_code == 200 - srjson = svg_response.json() - assert srjson["status"].lower() != "failure" - assert srjson["task_id"] is not None + else: + # test sync + route += "/img" + response = client.post(route, json=payload) + assert response.status_code == 200, response @pytest.mark.parametrize( @@ -242,8 +254,14 @@ def test_get_solution_sequence( @pytest.mark.parametrize("min_branch_size", [6]) @pytest.mark.parametrize("n_graphs", [1, 3]) @pytest.mark.parametrize("min_max", [(3, 4), (10, 11), (20, 40)]) +@pytest.mark.parametrize("timeout", [0.000001, 5.0]) def test_get_render_solution_sequence( - client, solution_sequence_response, min_branch_size, n_graphs, min_max + client, + solution_sequence_response, + min_branch_size, + n_graphs, + min_max, + timeout, ): key = min_branch_size, n_graphs, min_max post_response = solution_sequence_response[key] @@ -253,14 +271,15 @@ def test_get_render_solution_sequence( result_route = prjson.get("result_route") if result_route: - response = client.get(result_route + "/img") - assert response.status_code == 200 + q = f"?timeout={timeout:0.9f}" if timeout else "" + response = client.get(result_route + "/img" + q) + assert response.status_code == 200, response if "html" not in response.headers["content-type"]: # pragma: no branch srjson = response.json() - assert srjson["status"].lower() != "failure" + assert srjson["status"].lower() != "failure", srjson assert srjson["task_id"] is not None - response = poll_testclient_url(client, result_route + "/img") + response = poll_testclient_url(client, result_route + "/img" + q) assert "DOCTYPE svg PUBLIC" in response.content.decode() diff --git a/nereid/nereid/tests/test_src/test_network/test_algorithms.py b/nereid/nereid/tests/test_src/test_network/test_algorithms.py index 3bd4dcc5..86f07424 100644 --- a/nereid/nereid/tests/test_src/test_network/test_algorithms.py +++ b/nereid/nereid/tests/test_src/test_network/test_algorithms.py @@ -33,7 +33,7 @@ def test_network_algo_get_all_predecessors(g): for node in g.nodes(): nix = tsort.index(node) preds = get_all_predecessors(g, node) - ancestors = set(nx.ancestors(g, node)) + ancestors = nx.ancestors(g, node) # check equivalence with networkx assert preds == ancestors diff --git a/nereid/nereid/tests/test_src/test_network/test_tasks.py b/nereid/nereid/tests/test_src/test_network/test_tasks.py index 95e374f1..ba5ef592 100644 --- a/nereid/nereid/tests/test_src/test_network/test_tasks.py +++ b/nereid/nereid/tests/test_src/test_network/test_tasks.py @@ -4,11 +4,11 @@ from nereid.src.network.utils import graph_factory try: + import fast_sugiyama import matplotlib - import pydot except ImportError: # pragma: no cover matplotlib = None - pydot = None + fast_sugiyama = None @pytest.mark.parametrize( @@ -82,7 +82,9 @@ def test_network_subgraph_result(subgraph_result): @pytest.mark.skipif(matplotlib is None, reason="optional matplotlib is not installed") -@pytest.mark.skipif(pydot is None, reason="optional pydot is not installed") +@pytest.mark.skipif( + fast_sugiyama is None, reason="optional fast_sugiyama is not installed" +) def test_render_subgraph_svg(subgraph_result): result = tasks.render_subgraph_svg(subgraph_result).decode() assert "svg" in result @@ -102,7 +104,9 @@ def test_solution_sequence_result(solution_sequence_result): @pytest.mark.skipif(matplotlib is None, reason="optional matplotlib is not installed") -@pytest.mark.skipif(pydot is None, reason="optional pydot is not installed") +@pytest.mark.skipif( + fast_sugiyama is None, reason="optional fast_sugiyama is not installed" +) def test_render_solution_sequence_svg(solution_sequence_result): result = tasks.render_solution_sequence_svg(solution_sequence_result).decode() assert "svg" in result diff --git a/nereid/nereid/tests/test_src/test_network/test_validate.py b/nereid/nereid/tests/test_src/test_network/test_validate.py index cdde4ff1..0b31b0e2 100644 --- a/nereid/nereid/tests/test_src/test_network/test_validate.py +++ b/nereid/nereid/tests/test_src/test_network/test_validate.py @@ -46,7 +46,7 @@ def test_validate_network(edgelist, isvalid, result): (nx.gnc_graph(10, seed=42), False), # multiple out connections, cycles, duplicated edges (nx.random_k_out_graph(10, 2, 1, seed=42), False), - (nx.gn_graph(10, seed=42), True), + (nx.gn_graph(10, kernel=lambda _: 1, seed=42), True), ], ) def test_isvalid(g, expected): diff --git a/nereid/nereid/tests/test_src/test_watershed/conftest.py b/nereid/nereid/tests/test_src/test_watershed/conftest.py index 2943a036..732dd691 100644 --- a/nereid/nereid/tests/test_src/test_watershed/conftest.py +++ b/nereid/nereid/tests/test_src/test_watershed/conftest.py @@ -8,7 +8,7 @@ @pytest.fixture def watershed_graph(): - g = nx.gnr_graph(n=13, p=0.0, seed=0) + g = nx.gn_graph(n=13, kernel=lambda _: 1, seed=0) nx.relabel_nodes(g, lambda x: str(x), copy=False) # type: ignore return g diff --git a/nereid/nereid/tests/test_src/test_watershed/test_solve_watershed.py b/nereid/nereid/tests/test_src/test_watershed/test_solve_watershed.py index 23e4594f..1b9b1e3d 100644 --- a/nereid/nereid/tests/test_src/test_watershed/test_solve_watershed.py +++ b/nereid/nereid/tests/test_src/test_watershed/test_solve_watershed.py @@ -355,7 +355,7 @@ def test_solve_watershed_stable_with_subsets( def test_facility_load_reduction(contexts, tmnt_facility, dwf_override): context = contexts["default"] - g = nx.relabel_nodes(nx.gnr_graph(n=3, p=0.0, seed=0), lambda x: str(x)) # type: ignore + g = nx.relabel_nodes(nx.gn_graph(n=3, kernel=lambda _: 1, seed=0), lambda x: str(x)) # type: ignore data = { "2": { "area_acres": 9.58071049103565, diff --git a/nereid/nereid/tests/utils.py b/nereid/nereid/tests/utils.py index d6a1c681..75d2dc8f 100644 --- a/nereid/nereid/tests/utils.py +++ b/nereid/nereid/tests/utils.py @@ -24,7 +24,7 @@ def poll_testclient_url(testclient, url, timeout=5, verbose=False): # pragma: n tries = 0 t = 0 inc = 0.1 - + start = time.perf_counter() while t < timeout: tries += 1 if verbose: @@ -43,10 +43,12 @@ def poll_testclient_url(testclient, url, timeout=5, verbose=False): # pragma: n ) if status.lower() == "success": return response - t += inc + t = time.perf_counter() - start time.sleep(inc) - raise TimeoutError(f"Task did not return in time. Tried {tries} times") + raise TimeoutError( + f"Task did not return in time. Tried {tries} times in {t} seconds." + ) def is_equal_subset(subset: dict | list | set, superset: dict | list | set) -> bool: @@ -69,9 +71,11 @@ def is_equal_subset(subset: dict | list | set, superset: dict | list | set) -> b def check_graph_data_equal(g, subg): + # TODO: merge with similar logic in check_subgraph_response_equal + unstable_cols = ["_is_leaf", "_visited_at"] for node, dct in subg.nodes(data=True): for k, v in dct.items(): - if k == "_is_leaf": + if k in unstable_cols: continue og = g.nodes[node][k] if isinstance(v, str): @@ -96,8 +100,8 @@ def generate_n_random_valid_watershed_graphs( numpy.random.seed(seed) for i in range(n_graphs): n_nodes = numpy.random.randint(min_graph_nodes, max_graph_nodes) - offset = len(G.nodes()) - g = nx.gnr_graph(n_nodes, 0.0, seed=i) + offset = len(G.nodes) + g = nx.gn_graph(n_nodes, kernel=lambda _: 1, seed=i) G.add_edges_from([((offset + s), (offset + t)) for s, t in g.edges]) return G @@ -388,11 +392,12 @@ def generate_random_watershed_solve_request( def check_subgraph_response_equal(subgraph_results, original_results): + unstable_cols = ["_is_leaf", "_visited_at"] for subg_result in subgraph_results: node = subg_result["node_id"] og_result = [n for n in original_results if n["node_id"] == node][0] for k, v in subg_result.items(): - if k == "_is_leaf": + if k in unstable_cols: continue og = og_result[k] if isinstance(v, str): diff --git a/pyproject.toml b/pyproject.toml index 9c259a10..c540c620 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,7 @@ where = ["nereid"] "*" = ["_no_git*"] [project.optional-dependencies] -extras = ["graphviz", "matplotlib", "pydot"] +extras = ["matplotlib", "fast_sugiyama"] async-worker = ["celery[redis]", "nereid-engine[extras]"] base-app = [ "fastapi", @@ -68,7 +68,14 @@ base-app = [ "websockets", ] app = ["nereid-engine[async-worker]", "nereid-engine[base-app]"] -test = ["coverage", "httpx", "pytest", "pytest-cov", "pytest-xdist"] +test = [ + "nereid-engine[extras]", + "coverage", + "httpx", + "pytest", + "pytest-cov", + "pytest-xdist", +] dev = [ "nereid-engine[app]", "nereid-engine[test]", @@ -125,5 +132,6 @@ module = [ "pandas", "scipy.interpolate", "pydantic_settings", + "fast_sugiyama", ] ignore_missing_imports = true diff --git a/requirements/requirements_app_unpinned.txt b/requirements/requirements_app_unpinned.txt index d1444b8c..c8333b7c 100644 --- a/requirements/requirements_app_unpinned.txt +++ b/requirements/requirements_app_unpinned.txt @@ -4,3 +4,4 @@ uvicorn uvloop websockets brotli-asgi +watchfiles diff --git a/requirements/requirements_base_extras_unpinned.txt b/requirements/requirements_base_extras_unpinned.txt deleted file mode 100644 index 0c64eb04..00000000 --- a/requirements/requirements_base_extras_unpinned.txt +++ /dev/null @@ -1,4 +0,0 @@ -graphviz -matplotlib -pydot -watchfiles diff --git a/requirements/requirements_base_unpinned.txt b/requirements/requirements_base_unpinned.txt index 49669f75..ad4e5dea 100644 --- a/requirements/requirements_base_unpinned.txt +++ b/requirements/requirements_base_unpinned.txt @@ -1,8 +1,9 @@ +matplotlib networkx>=3.4,<3.5 +fast-sugiyama>=0.5 orjson pandas Pint -pyarrow pydantic>=2.4,<3.0 pydantic_settings python-dotenv diff --git a/requirements/requirements_dev.txt b/requirements/requirements_dev.txt index da6bb5b4..9a9964f9 100644 --- a/requirements/requirements_dev.txt +++ b/requirements/requirements_dev.txt @@ -1,29 +1,29 @@ # This file was autogenerated by uv via the following command: -# uv pip compile nereid/requirements/requirements_dev_unpinned.txt +# uv pip compile requirements/requirements_dev_unpinned.txt amqp==5.3.1 # via kombu annotated-types==0.7.0 # via pydantic -anyio==4.9.0 +anyio==4.11.0 # via # httpx # starlette # watchfiles -billiard==4.2.1 +billiard==4.2.2 # via celery brotli==1.1.0 # via brotli-asgi brotli-asgi==1.4.0 - # via -r nereid/requirements/requirements_app_unpinned.txt -celery==5.5.1 - # via -r nereid/requirements/requirements_app_async_unpinned.txt -certifi==2025.1.31 + # via -r requirements/requirements_app_unpinned.txt +celery==5.5.3 + # via -r requirements/requirements_app_async_unpinned.txt +certifi==2025.10.5 # via # httpcore # httpx cfgv==3.4.0 # via pre-commit -click==8.1.8 +click==8.3.0 # via # celery # click-didyoumean @@ -32,43 +32,43 @@ click==8.1.8 # uvicorn click-didyoumean==0.3.1 # via celery -click-plugins==1.1.1 +click-plugins==1.1.1.2 # via celery click-repl==0.3.0 # via celery -contourpy==1.3.2 +contourpy==1.3.3 # via matplotlib -coverage==7.8.0 +coverage==7.10.7 # via - # -r nereid/requirements/requirements_dev_unpinned.txt + # -r requirements/requirements_dev_unpinned.txt # pytest-cov cycler==0.12.1 # via matplotlib -distlib==0.3.9 +distlib==0.4.0 # via virtualenv execnet==2.1.1 # via pytest-xdist -fastapi==0.115.12 - # via -r nereid/requirements/requirements_app_unpinned.txt -filelock==3.18.0 +fast-sugiyama==0.5.0 + # via -r requirements/requirements_base_unpinned.txt +fastapi==0.118.3 + # via -r requirements/requirements_app_unpinned.txt +filelock==3.20.0 # via virtualenv flexcache==0.3 # via pint flexparser==0.4 # via pint -fonttools==4.57.0 +fonttools==4.60.1 # via matplotlib -graphviz==0.20.3 - # via -r nereid/requirements/requirements_base_extras_unpinned.txt -h11==0.14.0 +h11==0.16.0 # via # httpcore # uvicorn -httpcore==1.0.8 +httpcore==1.0.9 # via httpx httpx==0.28.1 - # via -r nereid/requirements/requirements_dev_unpinned.txt -identify==2.6.9 + # via -r requirements/requirements_dev_unpinned.txt +identify==2.6.15 # via pre-commit idna==3.10 # via @@ -77,111 +77,112 @@ idna==3.10 iniconfig==2.1.0 # via pytest jinja2==3.1.6 - # via -r nereid/requirements/requirements_app_unpinned.txt -kiwisolver==1.4.8 + # via -r requirements/requirements_app_unpinned.txt +kiwisolver==1.4.9 # via matplotlib -kombu==5.5.3 +kombu==5.5.4 # via celery -markupsafe==3.0.2 +markupsafe==3.0.3 # via jinja2 -matplotlib==3.10.1 - # via -r nereid/requirements/requirements_base_extras_unpinned.txt -mypy==1.15.0 - # via -r nereid/requirements/requirements_lint_unpinned.txt -mypy-extensions==1.0.0 +matplotlib==3.10.7 + # via -r requirements/requirements_base_unpinned.txt +mypy==1.18.2 + # via -r requirements/requirements_lint_unpinned.txt +mypy-extensions==1.1.0 # via mypy networkx==3.4.2 - # via -r nereid/requirements/requirements_base_unpinned.txt + # via -r requirements/requirements_base_unpinned.txt nodeenv==1.9.1 # via pre-commit -numpy==2.2.4 +numpy==2.3.3 # via # contourpy # matplotlib # pandas # scipy -orjson==3.10.16 - # via -r nereid/requirements/requirements_base_unpinned.txt -packaging==24.2 +orjson==3.11.3 + # via -r requirements/requirements_base_unpinned.txt +packaging==25.0 # via + # kombu # matplotlib # pytest -pandas==2.2.3 - # via -r nereid/requirements/requirements_base_unpinned.txt -pillow==11.2.1 +pandas==2.3.3 + # via -r requirements/requirements_base_unpinned.txt +pathspec==0.12.1 + # via mypy +pillow==11.3.0 # via matplotlib -pint==0.24.4 - # via -r nereid/requirements/requirements_base_unpinned.txt -platformdirs==4.3.7 +pint==0.25 + # via -r requirements/requirements_base_unpinned.txt +platformdirs==4.5.0 # via # pint # virtualenv -pluggy==1.5.0 - # via pytest -pre-commit==4.2.0 - # via -r nereid/requirements/requirements_lint_unpinned.txt -prompt-toolkit==3.0.51 +pluggy==1.6.0 + # via + # pytest + # pytest-cov +pre-commit==4.3.0 + # via -r requirements/requirements_lint_unpinned.txt +prompt-toolkit==3.0.52 # via click-repl -pyarrow==19.0.1 - # via -r nereid/requirements/requirements_base_unpinned.txt -pydantic==2.11.3 +pydantic==2.12.0 # via - # -r nereid/requirements/requirements_base_unpinned.txt + # -r requirements/requirements_base_unpinned.txt # fastapi # pydantic-settings -pydantic-core==2.33.1 +pydantic-core==2.41.1 # via pydantic -pydantic-settings==2.8.1 - # via -r nereid/requirements/requirements_base_unpinned.txt -pydot==3.0.4 - # via -r nereid/requirements/requirements_base_extras_unpinned.txt -pyparsing==3.2.3 - # via - # matplotlib - # pydot -pytest==8.3.5 +pydantic-settings==2.11.0 + # via -r requirements/requirements_base_unpinned.txt +pygments==2.19.2 + # via pytest +pyparsing==3.2.5 + # via matplotlib +pytest==8.4.2 # via - # -r nereid/requirements/requirements_dev_unpinned.txt + # -r requirements/requirements_dev_unpinned.txt # pytest-cov # pytest-xdist -pytest-cov==6.1.1 - # via -r nereid/requirements/requirements_dev_unpinned.txt -pytest-xdist==3.6.1 - # via -r nereid/requirements/requirements_dev_unpinned.txt +pytest-cov==7.0.0 + # via -r requirements/requirements_dev_unpinned.txt +pytest-xdist==3.8.0 + # via -r requirements/requirements_dev_unpinned.txt python-dateutil==2.9.0.post0 # via # celery # matplotlib # pandas -python-dotenv==1.1.0 +python-dotenv==1.1.1 # via - # -r nereid/requirements/requirements_base_unpinned.txt + # -r requirements/requirements_base_unpinned.txt # pydantic-settings pytz==2025.2 # via - # -r nereid/requirements/requirements_base_unpinned.txt + # -r requirements/requirements_base_unpinned.txt # pandas -pyyaml==6.0.2 +pyyaml==6.0.3 # via - # -r nereid/requirements/requirements_base_unpinned.txt + # -r requirements/requirements_base_unpinned.txt # pre-commit redis==5.2.1 - # via celery -ruff==0.11.6 - # via -r nereid/requirements/requirements_lint_unpinned.txt -scipy==1.15.2 - # via -r nereid/requirements/requirements_base_unpinned.txt + # via kombu +ruff==0.14.0 + # via -r requirements/requirements_lint_unpinned.txt +scipy==1.16.2 + # via -r requirements/requirements_base_unpinned.txt six==1.17.0 # via python-dateutil sniffio==1.3.1 # via anyio -starlette==0.46.2 +starlette==0.48.0 # via # brotli-asgi # fastapi tenacity==9.1.2 - # via -r nereid/requirements/requirements_app_async_unpinned.txt -typing-extensions==4.13.2 + # via -r requirements/requirements_app_async_unpinned.txt +typing-extensions==4.15.0 # via # anyio # fastapi @@ -191,27 +192,30 @@ typing-extensions==4.13.2 # pint # pydantic # pydantic-core + # starlette # typing-inspection -typing-inspection==0.4.0 - # via pydantic +typing-inspection==0.4.2 + # via + # pydantic + # pydantic-settings tzdata==2025.2 # via # kombu # pandas -uvicorn==0.34.1 - # via -r nereid/requirements/requirements_app_unpinned.txt +uvicorn==0.37.0 + # via -r requirements/requirements_app_unpinned.txt uvloop==0.21.0 - # via -r nereid/requirements/requirements_app_unpinned.txt + # via -r requirements/requirements_app_unpinned.txt vine==5.1.0 # via # amqp # celery # kombu -virtualenv==20.30.0 +virtualenv==20.34.0 # via pre-commit -watchfiles==1.0.5 - # via -r nereid/requirements/requirements_base_extras_unpinned.txt -wcwidth==0.2.13 +watchfiles==1.1.0 + # via -r requirements/requirements_app_unpinned.txt +wcwidth==0.2.14 # via prompt-toolkit websockets==15.0.1 - # via -r nereid/requirements/requirements_app_unpinned.txt + # via -r requirements/requirements_app_unpinned.txt diff --git a/requirements/requirements_lint.txt b/requirements/requirements_lint.txt index 02a50113..78c11ad2 100644 --- a/requirements/requirements_lint.txt +++ b/requirements/requirements_lint.txt @@ -1,28 +1,30 @@ # This file was autogenerated by uv via the following command: -# uv pip compile nereid/requirements/requirements_lint_unpinned.txt +# uv pip compile requirements/requirements_lint_unpinned.txt cfgv==3.4.0 # via pre-commit -distlib==0.3.9 +distlib==0.4.0 # via virtualenv -filelock==3.18.0 +filelock==3.20.0 # via virtualenv -identify==2.6.9 +identify==2.6.15 # via pre-commit -mypy==1.15.0 - # via -r nereid/requirements/requirements_lint_unpinned.txt -mypy-extensions==1.0.0 +mypy==1.18.2 + # via -r requirements/requirements_lint_unpinned.txt +mypy-extensions==1.1.0 # via mypy nodeenv==1.9.1 # via pre-commit -platformdirs==4.3.7 +pathspec==0.12.1 + # via mypy +platformdirs==4.5.0 # via virtualenv -pre-commit==4.2.0 - # via -r nereid/requirements/requirements_lint_unpinned.txt -pyyaml==6.0.2 +pre-commit==4.3.0 + # via -r requirements/requirements_lint_unpinned.txt +pyyaml==6.0.3 # via pre-commit -ruff==0.11.6 - # via -r nereid/requirements/requirements_lint_unpinned.txt -typing-extensions==4.13.2 +ruff==0.14.0 + # via -r requirements/requirements_lint_unpinned.txt +typing-extensions==4.15.0 # via mypy -virtualenv==20.30.0 +virtualenv==20.34.0 # via pre-commit diff --git a/requirements/requirements_nereid.txt b/requirements/requirements_nereid.txt index 6475d2c4..08bf3574 100644 --- a/requirements/requirements_nereid.txt +++ b/requirements/requirements_nereid.txt @@ -1,22 +1,22 @@ # This file was autogenerated by uv via the following command: -# uv pip compile nereid/requirements/requirements_nereid_unpinned.txt +# uv pip compile requirements/requirements_nereid_unpinned.txt amqp==5.3.1 # via kombu annotated-types==0.7.0 # via pydantic -anyio==4.9.0 +anyio==4.11.0 # via # starlette # watchfiles -billiard==4.2.1 +billiard==4.2.2 # via celery brotli==1.1.0 # via brotli-asgi brotli-asgi==1.4.0 - # via -r nereid/requirements/requirements_app_unpinned.txt -celery==5.5.1 - # via -r nereid/requirements/requirements_app_async_unpinned.txt -click==8.1.8 + # via -r requirements/requirements_app_unpinned.txt +celery==5.5.3 + # via -r requirements/requirements_app_async_unpinned.txt +click==8.3.0 # via # celery # click-didyoumean @@ -25,107 +25,103 @@ click==8.1.8 # uvicorn click-didyoumean==0.3.1 # via celery -click-plugins==1.1.1 +click-plugins==1.1.1.2 # via celery click-repl==0.3.0 # via celery -contourpy==1.3.2 +contourpy==1.3.3 # via matplotlib cycler==0.12.1 # via matplotlib -fastapi==0.115.12 - # via -r nereid/requirements/requirements_app_unpinned.txt +fast-sugiyama==0.5.0 + # via -r requirements/requirements_base_unpinned.txt +fastapi==0.118.3 + # via -r requirements/requirements_app_unpinned.txt flexcache==0.3 # via pint flexparser==0.4 # via pint -fonttools==4.57.0 +fonttools==4.60.1 # via matplotlib -graphviz==0.20.3 - # via -r nereid/requirements/requirements_base_extras_unpinned.txt -h11==0.14.0 +h11==0.16.0 # via uvicorn idna==3.10 # via anyio jinja2==3.1.6 - # via -r nereid/requirements/requirements_app_unpinned.txt -kiwisolver==1.4.8 + # via -r requirements/requirements_app_unpinned.txt +kiwisolver==1.4.9 # via matplotlib -kombu==5.5.3 +kombu==5.5.4 # via celery -markupsafe==3.0.2 +markupsafe==3.0.3 # via jinja2 -matplotlib==3.10.1 - # via -r nereid/requirements/requirements_base_extras_unpinned.txt +matplotlib==3.10.7 + # via -r requirements/requirements_base_unpinned.txt networkx==3.4.2 - # via -r nereid/requirements/requirements_base_unpinned.txt -numpy==2.2.4 + # via -r requirements/requirements_base_unpinned.txt +numpy==2.3.3 # via # contourpy # matplotlib # pandas # scipy -orjson==3.10.16 - # via -r nereid/requirements/requirements_base_unpinned.txt -packaging==24.2 - # via matplotlib -pandas==2.2.3 - # via -r nereid/requirements/requirements_base_unpinned.txt -pillow==11.2.1 +orjson==3.11.3 + # via -r requirements/requirements_base_unpinned.txt +packaging==25.0 + # via + # kombu + # matplotlib +pandas==2.3.3 + # via -r requirements/requirements_base_unpinned.txt +pillow==11.3.0 # via matplotlib -pint==0.24.4 - # via -r nereid/requirements/requirements_base_unpinned.txt -platformdirs==4.3.7 +pint==0.25 + # via -r requirements/requirements_base_unpinned.txt +platformdirs==4.5.0 # via pint -prompt-toolkit==3.0.51 +prompt-toolkit==3.0.52 # via click-repl -pyarrow==19.0.1 - # via -r nereid/requirements/requirements_base_unpinned.txt -pydantic==2.11.3 +pydantic==2.12.0 # via - # -r nereid/requirements/requirements_base_unpinned.txt + # -r requirements/requirements_base_unpinned.txt # fastapi # pydantic-settings -pydantic-core==2.33.1 +pydantic-core==2.41.1 # via pydantic -pydantic-settings==2.8.1 - # via -r nereid/requirements/requirements_base_unpinned.txt -pydot==3.0.4 - # via -r nereid/requirements/requirements_base_extras_unpinned.txt -pyparsing==3.2.3 - # via - # matplotlib - # pydot +pydantic-settings==2.11.0 + # via -r requirements/requirements_base_unpinned.txt +pyparsing==3.2.5 + # via matplotlib python-dateutil==2.9.0.post0 # via # celery # matplotlib # pandas -python-dotenv==1.1.0 +python-dotenv==1.1.1 # via - # -r nereid/requirements/requirements_base_unpinned.txt + # -r requirements/requirements_base_unpinned.txt # pydantic-settings pytz==2025.2 # via - # -r nereid/requirements/requirements_base_unpinned.txt + # -r requirements/requirements_base_unpinned.txt # pandas -pyyaml==6.0.2 - # via -r nereid/requirements/requirements_base_unpinned.txt +pyyaml==6.0.3 + # via -r requirements/requirements_base_unpinned.txt redis==5.2.1 - # via celery -scipy==1.15.2 - # via -r nereid/requirements/requirements_base_unpinned.txt + # via kombu +scipy==1.16.2 + # via -r requirements/requirements_base_unpinned.txt six==1.17.0 # via python-dateutil sniffio==1.3.1 # via anyio -starlette==0.46.2 +starlette==0.48.0 # via # brotli-asgi # fastapi tenacity==9.1.2 - # via -r nereid/requirements/requirements_app_async_unpinned.txt -typing-extensions==4.13.2 + # via -r requirements/requirements_app_async_unpinned.txt +typing-extensions==4.15.0 # via # anyio # fastapi @@ -134,25 +130,28 @@ typing-extensions==4.13.2 # pint # pydantic # pydantic-core + # starlette # typing-inspection -typing-inspection==0.4.0 - # via pydantic +typing-inspection==0.4.2 + # via + # pydantic + # pydantic-settings tzdata==2025.2 # via # kombu # pandas -uvicorn==0.34.1 - # via -r nereid/requirements/requirements_app_unpinned.txt +uvicorn==0.37.0 + # via -r requirements/requirements_app_unpinned.txt uvloop==0.21.0 - # via -r nereid/requirements/requirements_app_unpinned.txt + # via -r requirements/requirements_app_unpinned.txt vine==5.1.0 # via # amqp # celery # kombu -watchfiles==1.0.5 - # via -r nereid/requirements/requirements_base_extras_unpinned.txt -wcwidth==0.2.13 +watchfiles==1.1.0 + # via -r requirements/requirements_app_unpinned.txt +wcwidth==0.2.14 # via prompt-toolkit websockets==15.0.1 - # via -r nereid/requirements/requirements_app_unpinned.txt + # via -r requirements/requirements_app_unpinned.txt diff --git a/requirements/requirements_worker.txt b/requirements/requirements_worker.txt index 79a191d1..25546699 100644 --- a/requirements/requirements_worker.txt +++ b/requirements/requirements_worker.txt @@ -1,16 +1,14 @@ # This file was autogenerated by uv via the following command: -# uv pip compile nereid/requirements/requirements_worker_unpinned.txt +# uv pip compile requirements/requirements_worker_unpinned.txt amqp==5.3.1 # via kombu annotated-types==0.7.0 # via pydantic -anyio==4.9.0 - # via watchfiles -billiard==4.2.1 +billiard==4.2.2 # via celery -celery==5.5.1 - # via -r nereid/requirements/requirements_app_async_unpinned.txt -click==8.1.8 +celery==5.5.3 + # via -r requirements/requirements_app_async_unpinned.txt +click==8.3.0 # via # celery # click-didyoumean @@ -18,104 +16,97 @@ click==8.1.8 # click-repl click-didyoumean==0.3.1 # via celery -click-plugins==1.1.1 +click-plugins==1.1.1.2 # via celery click-repl==0.3.0 # via celery -contourpy==1.3.2 +contourpy==1.3.3 # via matplotlib cycler==0.12.1 # via matplotlib +fast-sugiyama==0.5.0 + # via -r requirements/requirements_base_unpinned.txt flexcache==0.3 # via pint flexparser==0.4 # via pint -fonttools==4.57.0 +fonttools==4.60.1 # via matplotlib -graphviz==0.20.3 - # via -r nereid/requirements/requirements_base_extras_unpinned.txt -idna==3.10 - # via anyio -kiwisolver==1.4.8 +kiwisolver==1.4.9 # via matplotlib -kombu==5.5.3 +kombu==5.5.4 # via celery -matplotlib==3.10.1 - # via -r nereid/requirements/requirements_base_extras_unpinned.txt +matplotlib==3.10.7 + # via -r requirements/requirements_base_unpinned.txt networkx==3.4.2 - # via -r nereid/requirements/requirements_base_unpinned.txt -numpy==2.2.4 + # via -r requirements/requirements_base_unpinned.txt +numpy==2.3.3 # via # contourpy # matplotlib # pandas # scipy -orjson==3.10.16 - # via -r nereid/requirements/requirements_base_unpinned.txt -packaging==24.2 - # via matplotlib -pandas==2.2.3 - # via -r nereid/requirements/requirements_base_unpinned.txt -pillow==11.2.1 +orjson==3.11.3 + # via -r requirements/requirements_base_unpinned.txt +packaging==25.0 + # via + # kombu + # matplotlib +pandas==2.3.3 + # via -r requirements/requirements_base_unpinned.txt +pillow==11.3.0 # via matplotlib -pint==0.24.4 - # via -r nereid/requirements/requirements_base_unpinned.txt -platformdirs==4.3.7 +pint==0.25 + # via -r requirements/requirements_base_unpinned.txt +platformdirs==4.5.0 # via pint -prompt-toolkit==3.0.51 +prompt-toolkit==3.0.52 # via click-repl -pyarrow==19.0.1 - # via -r nereid/requirements/requirements_base_unpinned.txt -pydantic==2.11.3 +pydantic==2.12.0 # via - # -r nereid/requirements/requirements_base_unpinned.txt + # -r requirements/requirements_base_unpinned.txt # pydantic-settings -pydantic-core==2.33.1 +pydantic-core==2.41.1 # via pydantic -pydantic-settings==2.8.1 - # via -r nereid/requirements/requirements_base_unpinned.txt -pydot==3.0.4 - # via -r nereid/requirements/requirements_base_extras_unpinned.txt -pyparsing==3.2.3 - # via - # matplotlib - # pydot +pydantic-settings==2.11.0 + # via -r requirements/requirements_base_unpinned.txt +pyparsing==3.2.5 + # via matplotlib python-dateutil==2.9.0.post0 # via # celery # matplotlib # pandas -python-dotenv==1.1.0 +python-dotenv==1.1.1 # via - # -r nereid/requirements/requirements_base_unpinned.txt + # -r requirements/requirements_base_unpinned.txt # pydantic-settings pytz==2025.2 # via - # -r nereid/requirements/requirements_base_unpinned.txt + # -r requirements/requirements_base_unpinned.txt # pandas -pyyaml==6.0.2 - # via -r nereid/requirements/requirements_base_unpinned.txt +pyyaml==6.0.3 + # via -r requirements/requirements_base_unpinned.txt redis==5.2.1 - # via celery -scipy==1.15.2 - # via -r nereid/requirements/requirements_base_unpinned.txt + # via kombu +scipy==1.16.2 + # via -r requirements/requirements_base_unpinned.txt six==1.17.0 # via python-dateutil -sniffio==1.3.1 - # via anyio tenacity==9.1.2 - # via -r nereid/requirements/requirements_app_async_unpinned.txt -typing-extensions==4.13.2 + # via -r requirements/requirements_app_async_unpinned.txt +typing-extensions==4.15.0 # via - # anyio # flexcache # flexparser # pint # pydantic # pydantic-core # typing-inspection -typing-inspection==0.4.0 - # via pydantic +typing-inspection==0.4.2 + # via + # pydantic + # pydantic-settings tzdata==2025.2 # via # kombu @@ -125,7 +116,5 @@ vine==5.1.0 # amqp # celery # kombu -watchfiles==1.0.5 - # via -r nereid/requirements/requirements_base_extras_unpinned.txt -wcwidth==0.2.13 +wcwidth==0.2.14 # via prompt-toolkit diff --git a/requirements/requirements_worker_unpinned.txt b/requirements/requirements_worker_unpinned.txt index ec98ce1b..e88ce524 100644 --- a/requirements/requirements_worker_unpinned.txt +++ b/requirements/requirements_worker_unpinned.txt @@ -1,3 +1,2 @@ -r requirements_base_unpinned.txt --r requirements_base_extras_unpinned.txt -r requirements_app_async_unpinned.txt diff --git a/scripts/bump_deps.sh b/scripts/bump_deps.sh index 7e54aab3..b379fc5e 100644 --- a/scripts/bump_deps.sh +++ b/scripts/bump_deps.sh @@ -2,7 +2,7 @@ set -e -pip install -U "uv>=0.6,<0.7" +pip install -U "uv>=0.9,<0.10" # dev uv pip compile requirements/requirements_dev_unpinned.txt > requirements/requirements_dev.txt diff --git a/scripts/test.sh b/scripts/test.sh index 5bde9dfd..2eed79d0 100644 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -4,4 +4,4 @@ set -e set -x make init-test -docker compose exec nereid-tests pytest "$@" +docker compose exec nereid-tests pytest nereid/tests/ "$@"