From 7e7964b672be61942ea8e3743f201d48bd80009a Mon Sep 17 00:00:00 2001 From: John Children Date: Thu, 24 Jul 2025 20:28:48 +0100 Subject: [PATCH 1/6] feat: Use rust internals feat: Continue adding rust internals chore: Try to get the tests working chore: Try to get rust internals working chore: Get tests running chore: Remove extra print statements chore: Further debugging chore: Refactor rust code into modules chore: Fix a bug related to loop scoping chore: Fix many type errors and simplify API chore: More docs and type fixes chore: Fix up more code drift chore: fix more type issues chore: Fix up more of type issues --- docs/source/conf.py | 4 + examples/example_workers/auth_worker/uv.lock | 10 +- examples/example_workers/qsci_worker/uv.lock | 10 +- examples/example_workers/scipy_worker/uv.lock | 10 +- .../substitution_worker/uv.lock | 10 +- pyproject.toml | 11 + tierkreis/pyproject.toml | 1 + tierkreis/tests/cli/data/factorial | 2 +- tierkreis/tests/cli/data/sample_graph | 2 +- tierkreis/tests/cli/test_run_workflow.py | 2 +- tierkreis/tests/cli/test_tkr.py | 4 +- tierkreis/tests/controller/loop_graphdata.py | 3 +- .../tests/controller/partial_graphdata.py | 2 +- .../tests/controller/sample_graphdata.py | 2 +- .../tests/controller/test_eagerifelse.py | 2 +- tierkreis/tests/controller/test_graphdata.py | 5 +- .../controller/test_graphdata_storage.py | 106 ++- tierkreis/tests/controller/test_locs.py | 63 +- .../tests/controller/test_read_loop_trace.py | 2 +- tierkreis/tests/controller/test_resume.py | 2 +- tierkreis/tests/errors/failing_worker/uv.lock | 10 +- tierkreis/tests/executor/test_hpc_executor.py | 2 +- tierkreis/tierkreis/builder.py | 25 +- tierkreis/tierkreis/cli/run_workflow.py | 6 +- tierkreis/tierkreis/cli/tkr.py | 2 +- tierkreis/tierkreis/controller/__init__.py | 33 +- tierkreis/tierkreis/controller/data/core.py | 3 - tierkreis/tierkreis/controller/data/graph.py | 253 ------ .../tierkreis/controller/data/location.py | 119 +-- tierkreis/tierkreis/controller/data/models.py | 20 +- tierkreis/tierkreis/controller/start.py | 199 ++--- .../tierkreis/controller/storage/adjacency.py | 37 +- .../controller/storage/filestorage.py | 2 +- .../tierkreis/controller/storage/graphdata.py | 54 +- .../tierkreis/controller/storage/protocol.py | 43 +- .../tierkreis/controller/storage/walk.py | 145 ++-- tierkreis/tierkreis/graphs/fold.py | 2 +- tierkreis/tierkreis/storage.py | 8 +- tierkreis/tierkreis/worker/worker.py | 2 +- tierkreis_core/.gitignore | 72 ++ tierkreis_core/Cargo.lock | 375 +++++++++ tierkreis_core/Cargo.toml | 31 + tierkreis_core/pyproject.toml | 22 + .../python/tierkreis_core/__init__.py | 23 + .../_tierkreis_core/__init__.pyi | 3 + .../tierkreis_core/_tierkreis_core/graph.pyi | 158 ++++ .../_tierkreis_core/identifiers.pyi | 36 + .../_tierkreis_core/location.pyi | 56 ++ .../tierkreis_core/_tierkreis_core/value.pyi | 3 + .../python/tierkreis_core/aliases.py | 15 + .../python/tierkreis_core/aliases.pyi | 14 + .../python/tierkreis_core/nodes.pyi | 135 ++++ tierkreis_core/python/tierkreis_core/py.typed | 0 .../python/tierkreis_core/steps.pyi | 23 + .../rust/bin/tierkreis-core-stubs-gen.rs | 126 +++ tierkreis_core/rust/graph.rs | 746 ++++++++++++++++++ tierkreis_core/rust/identifiers.rs | 151 ++++ tierkreis_core/rust/lib.rs | 30 + tierkreis_core/rust/location.rs | 374 +++++++++ tierkreis_core/rust/value.rs | 33 + .../tierkreis_visualization/app.py | 3 +- .../tierkreis_visualization/data/eval.py | 77 +- .../tierkreis_visualization/data/graph.py | 28 +- .../tierkreis_visualization/data/loop.py | 9 +- .../tierkreis_visualization/data/map.py | 13 +- .../tierkreis_visualization/data/models.py | 4 +- .../tierkreis_visualization/routers/models.py | 2 +- .../routers/navigation.py | 19 + .../routers/workflows.py | 2 +- .../tierkreis_visualization/storage.py | 2 +- .../visualize_graph.py | 2 +- uv.lock | 11 +- 72 files changed, 3043 insertions(+), 773 deletions(-) create mode 100644 tierkreis_core/.gitignore create mode 100644 tierkreis_core/Cargo.lock create mode 100644 tierkreis_core/Cargo.toml create mode 100644 tierkreis_core/pyproject.toml create mode 100644 tierkreis_core/python/tierkreis_core/__init__.py create mode 100644 tierkreis_core/python/tierkreis_core/_tierkreis_core/__init__.pyi create mode 100644 tierkreis_core/python/tierkreis_core/_tierkreis_core/graph.pyi create mode 100644 tierkreis_core/python/tierkreis_core/_tierkreis_core/identifiers.pyi create mode 100644 tierkreis_core/python/tierkreis_core/_tierkreis_core/location.pyi create mode 100644 tierkreis_core/python/tierkreis_core/_tierkreis_core/value.pyi create mode 100644 tierkreis_core/python/tierkreis_core/aliases.py create mode 100644 tierkreis_core/python/tierkreis_core/aliases.pyi create mode 100644 tierkreis_core/python/tierkreis_core/nodes.pyi create mode 100644 tierkreis_core/python/tierkreis_core/py.typed create mode 100644 tierkreis_core/python/tierkreis_core/steps.pyi create mode 100644 tierkreis_core/rust/bin/tierkreis-core-stubs-gen.rs create mode 100644 tierkreis_core/rust/graph.rs create mode 100644 tierkreis_core/rust/identifiers.rs create mode 100644 tierkreis_core/rust/lib.rs create mode 100644 tierkreis_core/rust/location.rs create mode 100644 tierkreis_core/rust/value.rs create mode 100644 tierkreis_visualization/tierkreis_visualization/routers/navigation.py diff --git a/docs/source/conf.py b/docs/source/conf.py index 49a0c3a1a..e326f233f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -16,6 +16,7 @@ extensions = ["autodoc2", "myst_nb"] autodoc2_packages = [ "../../tierkreis/tierkreis", + "../../tierkreis_core/python/tierkreis_core", { "path": "../../tierkreis_workers/aer_worker/main.py", "module": "aer_worker", @@ -29,6 +30,9 @@ "module": "pytket_worker", }, ] +autodoc2_module_all_regexes = [ + r"tierkreis_core", +] autodoc2_hidden_objects = ["private"] templates_path = ["_templates"] diff --git a/examples/example_workers/auth_worker/uv.lock b/examples/example_workers/auth_worker/uv.lock index 79a626029..3b431c2ae 100644 --- a/examples/example_workers/auth_worker/uv.lock +++ b/examples/example_workers/auth_worker/uv.lock @@ -176,10 +176,14 @@ version = "2.0.9" source = { editable = "../../../tierkreis" } dependencies = [ { name = "pydantic" }, + { name = "tierkreis-core" }, ] [package.metadata] -requires-dist = [{ name = "pydantic", specifier = "~=2.5" }] +requires-dist = [ + { name = "pydantic", specifier = "~=2.5" }, + { name = "tierkreis-core", editable = "../../../tierkreis_core" }, +] [package.metadata.requires-dev] build = [{ name = "build", extras = ["uv"] }] @@ -188,6 +192,10 @@ dev = [ { name = "pytest" }, ] +[[package]] +name = "tierkreis-core" +source = { editable = "../../../tierkreis_core" } + [[package]] name = "typing-extensions" version = "4.14.0" diff --git a/examples/example_workers/qsci_worker/uv.lock b/examples/example_workers/qsci_worker/uv.lock index b7a7601ed..782b97a67 100644 --- a/examples/example_workers/qsci_worker/uv.lock +++ b/examples/example_workers/qsci_worker/uv.lock @@ -878,10 +878,14 @@ version = "2.0.9" source = { editable = "../../../tierkreis" } dependencies = [ { name = "pydantic" }, + { name = "tierkreis-core" }, ] [package.metadata] -requires-dist = [{ name = "pydantic", specifier = "~=2.5" }] +requires-dist = [ + { name = "pydantic", specifier = "~=2.5" }, + { name = "tierkreis-core", editable = "../../../tierkreis_core" }, +] [package.metadata.requires-dev] build = [{ name = "build", extras = ["uv"] }] @@ -890,6 +894,10 @@ dev = [ { name = "pytest" }, ] +[[package]] +name = "tierkreis-core" +source = { editable = "../../../tierkreis_core" } + [[package]] name = "typing-extensions" version = "4.15.0" diff --git a/examples/example_workers/scipy_worker/uv.lock b/examples/example_workers/scipy_worker/uv.lock index 7fcd82eab..5f9597b08 100644 --- a/examples/example_workers/scipy_worker/uv.lock +++ b/examples/example_workers/scipy_worker/uv.lock @@ -270,10 +270,14 @@ version = "2.0.9" source = { editable = "../../../tierkreis" } dependencies = [ { name = "pydantic" }, + { name = "tierkreis-core" }, ] [package.metadata] -requires-dist = [{ name = "pydantic", specifier = "~=2.5" }] +requires-dist = [ + { name = "pydantic", specifier = "~=2.5" }, + { name = "tierkreis-core", editable = "../../../tierkreis_core" }, +] [package.metadata.requires-dev] build = [{ name = "build", extras = ["uv"] }] @@ -282,6 +286,10 @@ dev = [ { name = "pytest" }, ] +[[package]] +name = "tierkreis-core" +source = { editable = "../../../tierkreis_core" } + [[package]] name = "typing-extensions" version = "4.15.0" diff --git a/examples/example_workers/substitution_worker/uv.lock b/examples/example_workers/substitution_worker/uv.lock index b18491fa0..2130ebd45 100644 --- a/examples/example_workers/substitution_worker/uv.lock +++ b/examples/example_workers/substitution_worker/uv.lock @@ -380,10 +380,14 @@ version = "2.0.9" source = { directory = "../../../tierkreis" } dependencies = [ { name = "pydantic" }, + { name = "tierkreis-core" }, ] [package.metadata] -requires-dist = [{ name = "pydantic", specifier = "~=2.5" }] +requires-dist = [ + { name = "pydantic", specifier = "~=2.5" }, + { name = "tierkreis-core", editable = "../../../tierkreis_core" }, +] [package.metadata.requires-dev] build = [{ name = "build", extras = ["uv"] }] @@ -392,6 +396,10 @@ dev = [ { name = "pytest" }, ] +[[package]] +name = "tierkreis-core" +source = { editable = "../../../tierkreis_core" } + [[package]] name = "typing-extensions" version = "4.15.0" diff --git a/pyproject.toml b/pyproject.toml index f72eeaba3..0c11e055f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,7 @@ [tool.uv.workspace] members = [ "tierkreis", + "tierkreis_core", "tierkreis_visualization", "tierkreis_workers/*worker", ] @@ -33,3 +34,13 @@ docs = [ "furo>=2025.7.19", ] examples = ["pyscf>=2.9.0"] + +[tool.uv.sources] +tierkreis_core = { workspace = true } +quantinuum-sphinx = { git = "https://github.com/CQCL/quantinuum-sphinx.git" } + +[tool.maturin] +module-name = "tierkreis_core._tierkreis_core" +python-source = "tierkreis_core/python/" +manifest-path = "tierkreis_core/Cargo.toml" +features = ["pyo3/extension-module"] diff --git a/tierkreis/pyproject.toml b/tierkreis/pyproject.toml index 3011342ab..9a9f5f878 100644 --- a/tierkreis/pyproject.toml +++ b/tierkreis/pyproject.toml @@ -15,6 +15,7 @@ authors = [ requires-python = ">=3.12" dependencies = [ "pydantic~=2.5", + "tierkreis_core", ] [dependency-groups] diff --git a/tierkreis/tests/cli/data/factorial b/tierkreis/tests/cli/data/factorial index 26bc34046..ca1f66736 100644 --- a/tierkreis/tests/cli/data/factorial +++ b/tierkreis/tests/cli/data/factorial @@ -1 +1 @@ -{"nodes": [{"value": -1, "outputs": {"value": 5}, "inputs": {}, "type": "const"}, {"value": 1, "outputs": {"value": 4}, "inputs": {}, "type": "const"}, {"name": "n", "outputs": {"n": 7}, "inputs": {}, "type": "input"}, {"name": "factorial", "outputs": {"factorial": 6}, "inputs": {}, "type": "input"}, {"function_name": "builtins.igt", "inputs": {"a": [2, "n"], "b": [1, "value"]}, "outputs": {"value": 9}, "type": "function"}, {"function_name": "builtins.iadd", "inputs": {"a": [0, "value"], "b": [2, "n"]}, "outputs": {"value": 6}, "type": "function"}, {"graph": [3, "factorial"], "inputs": {"n": [5, "value"], "factorial": [3, "factorial"]}, "outputs": {"factorial_output": 7}, "type": "eval"}, {"function_name": "builtins.itimes", "inputs": {"a": [2, "n"], "b": [6, "factorial_output"]}, "outputs": {"value": 9}, "type": "function"}, {"value": 1, "outputs": {"value": 9}, "inputs": {}, "type": "const"}, {"pred": [4, "value"], "if_true": [7, "value"], "if_false": [8, "value"], "outputs": {"value": 10}, "inputs": {}, "type": "ifelse"}, {"inputs": {"factorial_output": [9, "value"]}, "outputs": {}, "type": "output"}], "fixed_inputs": {}, "graph_inputs": ["factorial", "n"], "graph_output_idx": 10} \ No newline at end of file +{"nodes":[[{"Const":{"value":{"Int":-1}}},{"value":5}],[{"Const":{"value":{"Int":1}}},{"value":4}],[{"Input":{"name":"n"}},{"n":7}],[{"Input":{"name":"factorial"}},{"factorial":6}],[{"Func":{"name":"builtins.igt","inputs":{"a":{"Value":[2,"n"]},"b":{"Value":[1,"value"]}}}},{"value":9}],[{"Func":{"name":"builtins.iadd","inputs":{"a":{"Value":[0,"value"]},"b":{"Value":[2,"n"]}}}},{"value":6}],[{"Eval":{"body":{"Value":[3,"factorial"]},"inputs":{"n":{"Value":[5,"value"]},"factorial":{"Value":[3,"factorial"]}}}},{"factorial_output":7}],[{"Func":{"name":"builtins.itimes","inputs":{"a":{"Value":[2,"n"]},"b":{"Value":[6,"factorial_output"]}}}},{"value":9}],[{"Const":{"value":{"Int":1}}},{"value":9}],[{"IfElse":{"pred":[4,"value"],"if_true":[7,"value"],"if_false":[8,"value"]}},{"value":10}],[{"Output":{"inputs":{"factorial_output":[9,"value"]}}},{}]],"fixed_inputs":{},"graph_inputs":["n","factorial"],"output_idx":10} diff --git a/tierkreis/tests/cli/data/sample_graph b/tierkreis/tests/cli/data/sample_graph index 462528802..b2867aae6 100644 --- a/tierkreis/tests/cli/data/sample_graph +++ b/tierkreis/tests/cli/data/sample_graph @@ -1 +1 @@ -{"nodes": [{"value": 0, "outputs": {"value": 3}, "inputs": {}, "type": "const"}, {"value": 6, "outputs": {"value": 3}, "inputs": {}, "type": "const"}, {"value": {"nodes": [{"name": "doubler_input", "outputs": {"doubler_input": 3}, "inputs": {}, "type": "input"}, {"name": "intercept", "outputs": {"intercept": 4}, "inputs": {}, "type": "input"}, {"value": 2, "outputs": {"value": 3}, "inputs": {}, "type": "const"}, {"function_name": "builtins.itimes", "inputs": {"a": [0, "doubler_input"], "b": [2, "value"]}, "outputs": {"value": 4}, "type": "function"}, {"function_name": "builtins.iadd", "inputs": {"a": [3, "value"], "b": [1, "intercept"]}, "outputs": {"value": 5}, "type": "function"}, {"inputs": {"doubler_output": [4, "value"]}, "outputs": {}, "type": "output"}], "fixed_inputs": {}, "graph_inputs": ["doubler_input", "intercept"], "graph_output_idx": 5}, "outputs": {}, "inputs": {}, "type": "const"}, {"graph": [2, "value"], "inputs": {"doubler_input": [1, "value"], "intercept": [0, "value"]}, "outputs": {"doubler_output": 4}, "type": "eval"}, {"inputs": {"simple_eval_output": [3, "doubler_output"]}, "outputs": {}, "type": "output"}], "fixed_inputs": {}, "graph_inputs": [], "graph_output_idx": 4} \ No newline at end of file +{"nodes":[[{"Const":{"value":{"Int":0}}},{"value":3}],[{"Const":{"value":{"Int":6}}},{"value":3}],[{"Const":{"value":{"Graph":{"nodes":[[{"Input":{"name":"doubler_input"}},{"doubler_input":3}],[{"Input":{"name":"intercept"}},{"intercept":4}],[{"Const":{"value":{"Int":2}}},{"value":3}],[{"Func":{"name":"builtins.itimes","inputs":{"a":{"Value":[0,"doubler_input"]},"b":{"Value":[2,"value"]}}}},{"value":4}],[{"Func":{"name":"builtins.iadd","inputs":{"a":{"Value":[3,"value"]},"b":{"Value":[1,"intercept"]}}}},{"value":5}],[{"Output":{"inputs":{"doubler_output":[4,"value"]}}},{}]],"fixed_inputs":{},"graph_inputs":["doubler_input","intercept"],"output_idx":5}}}},{}],[{"Eval":{"body":{"Value":[2,"value"]},"inputs":{"doubler_input":{"Value":[1,"value"]},"intercept":{"Value":[0,"value"]}}}},{"doubler_output":4}],[{"Output":{"inputs":{"simple_eval_output":[3,"doubler_output"]}}},{}]],"fixed_inputs":{},"graph_inputs":[],"output_idx":4} diff --git a/tierkreis/tests/cli/test_run_workflow.py b/tierkreis/tests/cli/test_run_workflow.py index ab9b766f0..a0d5d7c70 100644 --- a/tierkreis/tests/cli/test_run_workflow.py +++ b/tierkreis/tests/cli/test_run_workflow.py @@ -6,10 +6,10 @@ from unittest import mock -from tierkreis.controller.data.graph import GraphData from tierkreis.cli.run_workflow import run_workflow from tests.controller.sample_graphdata import simple_eval from tierkreis.controller.data.types import ptype_from_bytes +from tierkreis_core import GraphData logger = logging.getLogger(__name__) diff --git a/tierkreis/tests/cli/test_tkr.py b/tierkreis/tests/cli/test_tkr.py index 92545f019..89aa36c7d 100644 --- a/tierkreis/tests/cli/test_tkr.py +++ b/tierkreis/tests/cli/test_tkr.py @@ -6,14 +6,15 @@ from uuid import UUID from tierkreis.cli.tkr import load_graph, _load_inputs, main -from tierkreis.controller.data.graph import GraphData from tierkreis.controller.data.types import PType from tierkreis.exceptions import TierkreisError +from tierkreis_core import GraphData from tests.controller.sample_graphdata import simple_eval simple_eval_graph = simple_eval() + graph_params = [ ("tests.controller.sample_graphdata:simple_eval", simple_eval_graph), ("tierkreis/tests/controller/sample_graphdata.py:simple_eval", simple_eval_graph), @@ -112,6 +113,7 @@ def test_load_inputs_invalid() -> None: "args,result", cli_params, ids=["simple_eval_cli", "factorial_cli"] ) def test_end_to_end(args: list[str], result: dict[str, bytes]) -> None: + print(simple_eval().model_dump_json()) with mock.patch.object(sys, "argv", args): main() for key, value in result.items(): diff --git a/tierkreis/tests/controller/loop_graphdata.py b/tierkreis/tests/controller/loop_graphdata.py index 46467e1cc..a26dca788 100644 --- a/tierkreis/tests/controller/loop_graphdata.py +++ b/tierkreis/tests/controller/loop_graphdata.py @@ -1,8 +1,9 @@ from typing import NamedTuple + import tierkreis.builtins.stubs as tkr_builtins from tierkreis.controller.data.core import EmptyModel +from tierkreis_core import GraphData from tierkreis.builder import GraphBuilder -from tierkreis.controller.data.graph import GraphData from tierkreis.models import TKR diff --git a/tierkreis/tests/controller/partial_graphdata.py b/tierkreis/tests/controller/partial_graphdata.py index c4db981cf..d3d79747a 100644 --- a/tierkreis/tests/controller/partial_graphdata.py +++ b/tierkreis/tests/controller/partial_graphdata.py @@ -1,4 +1,4 @@ -from tierkreis.controller.data.graph import GraphData +from tierkreis_core import GraphData def ternary_add() -> GraphData: diff --git a/tierkreis/tests/controller/sample_graphdata.py b/tierkreis/tests/controller/sample_graphdata.py index ee46a88a4..0a529a026 100644 --- a/tierkreis/tests/controller/sample_graphdata.py +++ b/tierkreis/tests/controller/sample_graphdata.py @@ -1,5 +1,5 @@ from tierkreis import Labels -from tierkreis.controller.data.graph import GraphData +from tierkreis_core import GraphData def doubler_plus() -> GraphData: diff --git a/tierkreis/tests/controller/test_eagerifelse.py b/tierkreis/tests/controller/test_eagerifelse.py index 4bfe0d6f3..055c5b568 100644 --- a/tierkreis/tests/controller/test_eagerifelse.py +++ b/tierkreis/tests/controller/test_eagerifelse.py @@ -14,7 +14,7 @@ from tierkreis.controller.executor.shell_executor import ShellExecutor from tierkreis.controller.executor.uv_executor import UvExecutor from tierkreis.controller.storage.filestorage import ControllerFileStorage -from tierkreis.controller.data.graph import GraphData +from tierkreis_core import GraphData def eagerifelse_long_running() -> GraphData: diff --git a/tierkreis/tests/controller/test_graphdata.py b/tierkreis/tests/controller/test_graphdata.py index fa9d0ac89..3b05d45e5 100644 --- a/tierkreis/tests/controller/test_graphdata.py +++ b/tierkreis/tests/controller/test_graphdata.py @@ -1,10 +1,9 @@ import pytest -from tierkreis.exceptions import TierkreisError -from tierkreis.controller.data.graph import GraphData +from tierkreis_core import GraphData def test_only_one_output(): - with pytest.raises(TierkreisError): + with pytest.raises(ValueError): g = GraphData() g.output({"one": g.const(1)}) g.output({"two": g.const(2)}) diff --git a/tierkreis/tests/controller/test_graphdata_storage.py b/tierkreis/tests/controller/test_graphdata_storage.py index 0d053a105..72e736a1c 100644 --- a/tierkreis/tests/controller/test_graphdata_storage.py +++ b/tierkreis/tests/controller/test_graphdata_storage.py @@ -1,33 +1,35 @@ from uuid import UUID import pytest from tests.controller.sample_graphdata import simple_eval, simple_map -from tierkreis.controller.data.core import PortID -from tierkreis.controller.data.graph import ( - Const, - Eval, - Func, - GraphData, - Input, - graph_node_from_loc, -) from tierkreis.controller.data.location import Loc from tierkreis.controller.storage.graphdata import GraphDataStorage from tierkreis.exceptions import TierkreisError +from tierkreis_core import ( + PortID, + GraphData, + NodeDef, + ExteriorRef, + NodeDescription, + ValueRef, + NodeIndex, +) @pytest.mark.parametrize( ["node_location_str", "graph", "target"], [ - ("-.N0", simple_eval(), Const(0, outputs={"value": 3})), - ("-.N4.M0", simple_map(), Eval((-1, "body"), {})), - ("-.N4.M0.N-1", simple_map(), Eval((-1, "body"), {})), + ("-.N0", simple_eval(), NodeDef.Const(0)), + ("-.N4.M0", simple_map(), NodeDef.Eval(ExteriorRef("body"), {})), + ("-.N4.M0.E", simple_map(), NodeDef.Eval(ExteriorRef("body"), {})), ], ) -def test_read_nodedef(node_location_str: str, graph: GraphData, target: str) -> None: +def test_read_nodedescription_definition( + node_location_str: str, graph: GraphData, target: NodeDef +) -> None: loc = Loc(node_location_str) storage = GraphDataStorage(UUID(int=0), graph) - node_def = storage.read_node_def(loc) - assert node_def == target + node_def = storage.read_node_description(loc) + assert node_def.definition == target @pytest.mark.parametrize( @@ -72,35 +74,75 @@ def test_read_output_ports( @pytest.mark.parametrize( ["node_location_str", "graph", "target"], [ - ("-.N0", simple_eval(), Const(0, outputs={"value": 3})), - ("-.N3.N1", simple_eval(), Input("intercept", outputs={"intercept": 4})), + ( + "-.N0", + simple_eval(), + NodeDescription(NodeDef.Const(0), outputs={"value": NodeIndex(3)}), + ), + ( + "-.N3.N1", + simple_eval(), + NodeDescription( + NodeDef.Input("intercept"), outputs={"intercept": NodeIndex(4)} + ), + ), ( "-.N3.N3", simple_eval(), - Func( - "builtins.itimes", - inputs={"a": (0, "doubler_input"), "b": (2, "value")}, - outputs={"value": 4}, + NodeDescription( + NodeDef.Func( + "builtins.itimes", + inputs={ + "a": ValueRef(NodeIndex(0), "doubler_input"), + "b": ValueRef(NodeIndex(2), "value"), + }, + ), + outputs={"value": NodeIndex(4)}, ), ), - ("-.N-1", simple_eval(), Eval((-1, "body"), {})), - ("-.N3.N-1", simple_eval(), Eval((-1, "body"), {})), + ( + "-.E", + simple_eval(), + NodeDescription(NodeDef.Eval(ExteriorRef("body"), {})), + ), + ( + "-.N3.E", + simple_eval(), + NodeDescription(NodeDef.Eval(ExteriorRef("body"), {})), + ), ( "-.N4.M0", simple_map(), - Eval( - (-1, "body"), - inputs={"doubler_input": (2, "*"), "intercept": (0, "value")}, - outputs={"*": 5}, + NodeDescription( + NodeDef.Eval( + ExteriorRef("body"), + inputs={ + "doubler_input": ValueRef(NodeIndex(2), "*"), + "intercept": ValueRef(NodeIndex(0), "value"), + }, + ), + outputs={"*": NodeIndex(5)}, + ), + ), + ( + "-.N4.M0.E", + simple_map(), + NodeDescription(NodeDef.Eval(ExteriorRef("body"), {})), + ), + ( + "-.N4.M0.N1", + simple_map(), + NodeDescription( + NodeDef.Input("intercept"), outputs={"intercept": NodeIndex(4)} ), ), - ("-.N4.M0.N-1", simple_map(), Eval((-1, "body"), {})), - ("-.N4.M0.N1", simple_map(), Input("intercept", outputs={"intercept": 4})), ], ) def test_graph_node_from_loc( - node_location_str: str, graph: GraphData, target: str + node_location_str: str, graph: GraphData, target: NodeDescription ) -> None: loc = Loc(node_location_str) - node_def, _ = graph_node_from_loc(loc, graph) - assert node_def == target + node_description = graph.query_node_description(loc) + # TOOD: Avoiding comparing parent graphs + assert node_description.definition == target.definition + assert node_description.outputs == target.outputs diff --git a/tierkreis/tests/controller/test_locs.py b/tierkreis/tests/controller/test_locs.py index 52051cf1f..348fa1f65 100644 --- a/tierkreis/tests/controller/test_locs.py +++ b/tierkreis/tests/controller/test_locs.py @@ -1,7 +1,6 @@ import pytest -from tierkreis.controller.data.location import Loc, NodeStep -from tierkreis.exceptions import TierkreisError +from tierkreis_core import NodeStep, Loc node_location_1 = Loc() node_location_1 = node_location_1.N(1) @@ -63,10 +62,10 @@ def test_parent(node_location: Loc, loc_str: str) -> None: @pytest.mark.parametrize( ["node_location", "node_step", "loc_str"], [ - (node_location_1, ("N", 1), "-.L0.N3.L2.N0.M7.N10"), - (node_location_2, ("N", 0), "-.L0.N3.N8.N0"), - (node_location_3, ("N", 0), "-"), - (node_location_4, "-", ""), + (node_location_1, NodeStep("N1"), "-.L0.N3.L2.N0.M7.N10"), + (node_location_2, NodeStep("N0"), "-.L0.N3.N8.N0"), + (node_location_3, NodeStep("N0"), "-"), + (node_location_4, NodeStep("-"), ""), ], ) def test_pop_first(node_location: Loc, node_step: NodeStep, loc_str: str) -> None: @@ -79,10 +78,10 @@ def test_pop_first(node_location: Loc, node_step: NodeStep, loc_str: str) -> Non @pytest.mark.parametrize( ["node_location", "node_step", "loc_str"], [ - (node_location_1, ("N", 10), "-.N1.L0.N3.L2.N0.M7"), - (node_location_2, ("N", 0), "-.N0.L0.N3.N8"), - (node_location_3, ("N", 0), "-"), - (node_location_4, "-", ""), + (node_location_1, NodeStep("N10"), "-.N1.L0.N3.L2.N0.M7"), + (node_location_2, NodeStep("N0"), "-.N0.L0.N3.N8"), + (node_location_3, NodeStep("N0"), "-"), + (node_location_4, NodeStep("-"), ""), ], ) def test_pop_last(node_location: Loc, node_step: NodeStep, loc_str: str) -> None: @@ -94,9 +93,9 @@ def test_pop_last(node_location: Loc, node_step: NodeStep, loc_str: str) -> None def test_pop_empty() -> None: loc = Loc("") - with pytest.raises(TierkreisError): + with pytest.raises(ValueError): loc.pop_first() - with pytest.raises(TierkreisError): + with pytest.raises(ValueError): loc.pop_last() @@ -104,27 +103,27 @@ def test_pop_first_multiple() -> None: loc = node_location_2 pop = loc.pop_first() (step, remainder) = pop - assert step == ("N", 0) + assert step == NodeStep("N0") assert remainder == Loc("-.L0.N3.N8.N0") pop = remainder.pop_first() (step, remainder) = pop - assert step == ("L", 0) + assert step == NodeStep("L0") assert remainder == Loc("-.N3.N8.N0") pop = remainder.pop_first() (step, remainder) = pop - assert step == ("N", 3) + assert step == NodeStep("N3") assert remainder == Loc("-.N8.N0") pop = remainder.pop_first() (step, remainder) = pop - assert step == ("N", 8) + assert step == NodeStep("N8") assert remainder == Loc("-.N0") pop = remainder.pop_first() (step, remainder) = pop - assert step == ("N", 0) + assert step == NodeStep("N0") assert remainder == Loc("-") pop = remainder.pop_first() (step, remainder) = pop - assert step == "-" + assert step == NodeStep("-") assert remainder == Loc("") @@ -132,42 +131,42 @@ def test_pop_last_multiple() -> None: loc = node_location_2 pop = loc.pop_last() (step, remainder) = pop - assert step == ("N", 0) + assert step == NodeStep("N0") assert remainder == Loc("-.N0.L0.N3.N8") pop = remainder.pop_last() (step, remainder) = pop - assert step == ("N", 8) + assert step == NodeStep("N8") assert remainder == Loc("-.N0.L0.N3") pop = remainder.pop_last() (step, remainder) = pop - assert step == ("N", 3) + assert step == NodeStep("N3") assert remainder == Loc("-.N0.L0") pop = remainder.pop_last() (step, remainder) = pop - assert step == ("L", 0) + assert step == NodeStep("L0") assert remainder == Loc("-.N0") pop = remainder.pop_last() (step, remainder) = pop - assert step == ("N", 0) + assert step == NodeStep("N0") assert remainder == Loc("-") pop = remainder.pop_last() (step, remainder) = pop - assert step == "-" + assert step == NodeStep("-") assert remainder == Loc("") @pytest.mark.parametrize( - ["node_location", "index"], + ["node_location", "expectation"], [ - (node_location_1, 10), - (node_location_2, 0), - (node_location_3, 0), - (node_location_4, 0), - (Loc().N(-1), -1), + (node_location_1, False), + (node_location_2, False), + (node_location_3, False), + (node_location_4, False), + (Loc().exterior(), True), ], ) -def test_get_last_index(node_location: Loc, index: int) -> None: - assert node_location.peek_index() == index +def test_last_step_exterior(node_location: Loc, expectation: bool) -> None: + assert node_location.last_step_exterior() == expectation @pytest.mark.parametrize( diff --git a/tierkreis/tests/controller/test_read_loop_trace.py b/tierkreis/tests/controller/test_read_loop_trace.py index f949d976e..9916b5fe7 100644 --- a/tierkreis/tests/controller/test_read_loop_trace.py +++ b/tierkreis/tests/controller/test_read_loop_trace.py @@ -9,8 +9,8 @@ from tierkreis.controller.executor.shell_executor import ShellExecutor from tierkreis.controller.storage.filestorage import ControllerFileStorage from tierkreis.controller.storage.in_memory import ControllerInMemoryStorage -from tierkreis.controller.data.graph import GraphData from tierkreis.storage import read_loop_trace +from tierkreis_core import GraphData return_value = [ diff --git a/tierkreis/tests/controller/test_resume.py b/tierkreis/tests/controller/test_resume.py index 38928fe12..5a9a6816b 100644 --- a/tierkreis/tests/controller/test_resume.py +++ b/tierkreis/tests/controller/test_resume.py @@ -35,8 +35,8 @@ from tierkreis.controller.executor.shell_executor import ShellExecutor from tierkreis.controller.storage.filestorage import ControllerFileStorage from tierkreis.controller.storage.in_memory import ControllerInMemoryStorage -from tierkreis.controller.data.graph import GraphData from tierkreis.storage import read_outputs +from tierkreis_core import GraphData param_data: list[tuple[GraphData, Any, str, dict[str, PType] | PType]] = [ (simple_eval(), {"simple_eval_output": 12}, "simple_eval", {}), diff --git a/tierkreis/tests/errors/failing_worker/uv.lock b/tierkreis/tests/errors/failing_worker/uv.lock index 17e80c40a..3374f0ebc 100644 --- a/tierkreis/tests/errors/failing_worker/uv.lock +++ b/tierkreis/tests/errors/failing_worker/uv.lock @@ -89,10 +89,14 @@ version = "2.0.9" source = { editable = "../../../" } dependencies = [ { name = "pydantic" }, + { name = "tierkreis-core" }, ] [package.metadata] -requires-dist = [{ name = "pydantic", specifier = "~=2.5" }] +requires-dist = [ + { name = "pydantic", specifier = "~=2.5" }, + { name = "tierkreis-core", editable = "../../../../tierkreis_core" }, +] [package.metadata.requires-dev] build = [{ name = "build", extras = ["uv"] }] @@ -101,6 +105,10 @@ dev = [ { name = "pytest" }, ] +[[package]] +name = "tierkreis-core" +source = { editable = "../../../../tierkreis_core" } + [[package]] name = "typing-extensions" version = "4.13.2" diff --git a/tierkreis/tests/executor/test_hpc_executor.py b/tierkreis/tests/executor/test_hpc_executor.py index 84b287887..506f8dc49 100644 --- a/tierkreis/tests/executor/test_hpc_executor.py +++ b/tierkreis/tests/executor/test_hpc_executor.py @@ -3,7 +3,6 @@ import pytest from tierkreis.builder import GraphBuilder from tierkreis.controller import run_graph -from tierkreis.controller.data.graph import GraphData from tierkreis.controller.data.models import TKR from tierkreis.controller.executor.hpc.job_spec import ( JobSpec, @@ -12,6 +11,7 @@ ) from tierkreis.controller.executor.hpc.slurm import SLURMExecutor from tierkreis.controller.storage.filestorage import ControllerFileStorage +from tierkreis_core import GraphData from tests.executor.stubs import mpi_rank_info from tierkreis.storage import read_outputs diff --git a/tierkreis/tierkreis/builder.py b/tierkreis/tierkreis/builder.py index 0d6969e64..76e04d101 100644 --- a/tierkreis/tierkreis/builder.py +++ b/tierkreis/tierkreis/builder.py @@ -12,7 +12,8 @@ init_tmodel, ) from tierkreis.controller.data.types import PType -from tierkreis.controller.data.graph import GraphData, ValueRef +from tierkreis_core import GraphData, ExteriorRef, ValueRef +import tierkreis_core @dataclass @@ -33,7 +34,7 @@ def out() -> type[Out]: ... @dataclass class TypedGraphRef[Ins: TModel, Outs: TModel]: - graph_ref: ValueRef + graph_ref: ExteriorRef | ValueRef outputs_type: type[Outs] inputs_type: type[Ins] @@ -77,12 +78,12 @@ def get_data(self) -> GraphData: return self.data def ref(self) -> TypedGraphRef[Inputs, Outputs]: - return TypedGraphRef((-1, "body"), self.outputs_type, self.inputs_type) + return TypedGraphRef(ExteriorRef("body"), self.outputs_type, self.inputs_type) def outputs(self, outputs: Outputs): self.data.output(inputs=dict_from_tmodel(outputs)) - def const[T: PType](self, value: T) -> TKR[T]: + def const[T: tierkreis_core.aliases.Value](self, value: T) -> TKR[T]: idx, port = self.data.const(value) return TKR[T](idx, port) @@ -105,9 +106,9 @@ def eifelse[A: PType, B: PType]( def _graph_const[A: TModel, B: TModel]( self, graph: "GraphBuilder[A, B]" ) -> TypedGraphRef[A, B]: - idx, port = self.data.const(graph.data.model_dump()) + idx, port = self.data.const(graph.data) return TypedGraphRef[A, B]( - graph_ref=(idx, port), + graph_ref=ValueRef(idx, port), outputs_type=graph.outputs_type, inputs_type=graph.inputs_type, ) @@ -117,7 +118,7 @@ def task[Out: TModel](self, f: Function[Out]) -> Out: ins = dict_from_tmodel(f) idx, _ = self.data.func(name, ins)("dummy") OutModel = f.out() - outputs = [(idx, x) for x in model_fields(OutModel)] + outputs = [ValueRef(idx, x) for x in model_fields(OutModel)] return init_tmodel(OutModel, outputs) @overload @@ -131,7 +132,7 @@ def eval[A: TModel, B: TModel]( body = self._graph_const(body) idx, _ = self.data.eval(body.graph_ref, dict_from_tmodel(a))("dummy") - outputs = [(idx, x) for x in model_fields(body.outputs_type)] + outputs = [ValueRef(idx, x) for x in model_fields(body.outputs_type)] return init_tmodel(body.outputs_type, outputs) @overload @@ -155,16 +156,16 @@ def loop[A: TModel, B: LoopOutput]( idx, _ = self.data.loop(g, dict_from_tmodel(a), "should_continue", name)( "dummy" ) - outputs = [(idx, x) for x in model_fields(body.outputs_type)] + outputs = [ValueRef(idx, x) for x in model_fields(body.outputs_type)] return init_tmodel(body.outputs_type, outputs) def _unfold_list[T: PType](self, ref: TKR[list[T]]) -> TList[TKR[T]]: - ins = (ref.node_index, ref.port_id) + ins = ValueRef(ref.node_index, ref.port_id) idx, _ = self.data.func("builtins.unfold_values", {"value": ins})("dummy") return TList(TKR[T](idx, "*")) def _fold_list[T: PType](self, refs: TList[TKR[T]]) -> TKR[list[T]]: - value_ref = (refs._value.node_index, refs._value.port_id) + value_ref = ValueRef(refs._value.node_index, refs._value.port_id) idx, _ = self.data.func("builtins.fold_values", {"values_glob": value_ref})( "dummy" ) @@ -187,7 +188,7 @@ def _map_graph_full[A: TModel, B: TModel]( ins = dict_from_tmodel(aes._value) idx, _ = self.data.map(body.graph_ref, ins)("x") - refs = [(idx, s + "-*") for s in model_fields(body.outputs_type)] + refs = [ValueRef(idx, s + "-*") for s in model_fields(body.outputs_type)] return TList(init_tmodel(body.outputs_type, refs)) @overload diff --git a/tierkreis/tierkreis/cli/run_workflow.py b/tierkreis/tierkreis/cli/run_workflow.py index 2446c73db..1abca0c51 100644 --- a/tierkreis/tierkreis/cli/run_workflow.py +++ b/tierkreis/tierkreis/cli/run_workflow.py @@ -3,12 +3,12 @@ import logging from tierkreis.controller import run_graph -from tierkreis.controller.data.graph import GraphData from tierkreis.controller.data.location import Loc from tierkreis.controller.data.types import PType from tierkreis.controller.storage.filestorage import ControllerFileStorage from tierkreis.controller.executor.shell_executor import ShellExecutor from tierkreis.controller.executor.uv_executor import UvExecutor +from tierkreis_core import GraphData logger = logging.getLogger(__name__) @@ -55,7 +55,7 @@ def run_workflow( n_iterations, polling_interval_seconds, ) - if print_output: - all_outputs = graph.nodes[graph.output_idx()].inputs + if print_output and (output_idx := graph.output_idx()): + all_outputs = graph.get_nodedef(output_idx).inputs for output in all_outputs: print(f"{output}: {storage.read_output(Loc(), output)}") diff --git a/tierkreis/tierkreis/cli/tkr.py b/tierkreis/tierkreis/cli/tkr.py index 7edc5eb7b..6451c1ee0 100644 --- a/tierkreis/tierkreis/cli/tkr.py +++ b/tierkreis/tierkreis/cli/tkr.py @@ -9,9 +9,9 @@ from typing import Any, Callable from tierkreis.cli.run_workflow import run_workflow -from tierkreis.controller.data.graph import GraphData from tierkreis.controller.data.types import PType, ptype_from_bytes from tierkreis.exceptions import TierkreisError +from tierkreis_core import GraphData def _import_from_path(module_name: str, file_path: str) -> Any: diff --git a/tierkreis/tierkreis/controller/__init__.py b/tierkreis/tierkreis/controller/__init__.py index 0a9c3d1d2..9fb5440ac 100644 --- a/tierkreis/tierkreis/controller/__init__.py +++ b/tierkreis/tierkreis/controller/__init__.py @@ -2,14 +2,13 @@ from time import sleep from tierkreis.builder import GraphBuilder -from tierkreis.controller.data.graph import Eval, GraphData from tierkreis.controller.data.location import Loc from tierkreis.controller.data.types import PType, bytes_from_ptype, ptype_from_bytes from tierkreis.controller.executor.protocol import ControllerExecutor from tierkreis.controller.start import NodeRunData, start, start_nodes from tierkreis.controller.storage.protocol import ControllerStorage from tierkreis.controller.storage.walk import walk_node -from tierkreis.controller.data.core import PortID, ValueRef +from tierkreis_core import ExteriorRef, GraphData, new_eval_root, PortID root_loc = Loc("") logger = logging.getLogger(__name__) @@ -32,16 +31,24 @@ def run_graph( if len(remaining_inputs) > 0: logger.warning(f"Some inputs were not provided: {remaining_inputs}") - storage.write_metadata(Loc("")) + storage.write_metadata(root_loc) for name, value in graph_inputs.items(): - storage.write_output(root_loc.N(-1), name, bytes_from_ptype(value)) + storage.write_output(root_loc.exterior(), name, bytes_from_ptype(value)) - storage.write_output(root_loc.N(-1), "body", bytes_from_ptype(g)) + storage.write_output(root_loc.exterior(), "body", bytes_from_ptype(g)) - inputs: dict[PortID, ValueRef] = { - k: (-1, k) for k, _ in graph_inputs.items() if k != "body" + inputs: dict[PortID, ExteriorRef] = { + k: ExteriorRef(k) for k in graph_inputs.keys() if k != "body" } - node_run_data = NodeRunData(Loc(), Eval((-1, "body"), inputs), []) + graph_outputs = g.graph_outputs() + if graph_outputs is None: + raise ValueError("Cannot run a graph with no outputs.") + + node_run_data = NodeRunData( + Loc(), + new_eval_root(inputs), + graph_outputs, + ) start(storage, executor, node_run_data) resume_graph(storage, executor, n_iterations, polling_interval_seconds) @@ -52,15 +59,19 @@ def resume_graph( n_iterations: int = 10000, polling_interval_seconds: float = 0.01, ) -> None: - message = storage.read_output(Loc().N(-1), "body") + message = storage.read_output(Loc().exterior(), "body") graph = ptype_from_bytes(message, GraphData) + output_idx = graph.output_idx() + if output_idx is None: + raise ValueError("Cannot resume a graph with no Output node.") + for _ in range(n_iterations): - walk_results = walk_node(storage, Loc(), graph.output_idx(), graph) + walk_results = walk_node(storage, Loc(), output_idx, graph) if walk_results.errored != []: # TODO: add to base class after storage refactor (storage.logs_path.parent / "-" / "_error").touch() - node_errors = "\n".join(x for x in walk_results.errored) + node_errors = "\n".join(str(x) for x in walk_results.errored) storage.write_node_errors(Loc(), node_errors) print("\n\nGraph finished with errors.\n\n") diff --git a/tierkreis/tierkreis/controller/data/core.py b/tierkreis/tierkreis/controller/data/core.py index 1aa3422f7..cfa297054 100644 --- a/tierkreis/tierkreis/controller/data/core.py +++ b/tierkreis/tierkreis/controller/data/core.py @@ -13,9 +13,6 @@ ) -PortID = str -NodeIndex = int -ValueRef = tuple[NodeIndex, PortID] SerializationFormat = Literal["bytes", "json", "unknown"] diff --git a/tierkreis/tierkreis/controller/data/graph.py b/tierkreis/tierkreis/controller/data/graph.py index 624dbd572..e69de29bb 100644 --- a/tierkreis/tierkreis/controller/data/graph.py +++ b/tierkreis/tierkreis/controller/data/graph.py @@ -1,253 +0,0 @@ -from dataclasses import dataclass, field -import logging -from typing import Any, Callable, Literal, assert_never -from pydantic import BaseModel, RootModel -from tierkreis.controller.data.core import PortID -from tierkreis.controller.data.core import NodeIndex -from tierkreis.controller.data.core import ValueRef -from tierkreis.controller.data.location import Loc, OutputLoc -from tierkreis.controller.data.types import PType, ptype_from_bytes -from tierkreis.exceptions import TierkreisError - -logger = logging.getLogger(__name__) - - -@dataclass -class Func: - function_name: str - inputs: dict[PortID, ValueRef] - outputs: dict[PortID, NodeIndex] = field(default_factory=lambda: {}) - type: Literal["function"] = field(default="function") - - -@dataclass -class Eval: - graph: ValueRef - inputs: dict[PortID, ValueRef] - outputs: dict[PortID, NodeIndex] = field(default_factory=lambda: {}) - type: Literal["eval"] = field(default="eval") - - -@dataclass -class Loop: - body: ValueRef - inputs: dict[PortID, ValueRef] - continue_port: PortID # The port that specifies if the loop should continue. - outputs: dict[PortID, NodeIndex] = field(default_factory=lambda: {}) - type: Literal["loop"] = field(default="loop") - name: str | None = None - - -@dataclass -class Map: - body: ValueRef - inputs: dict[PortID, ValueRef] - outputs: dict[PortID, NodeIndex] = field(default_factory=lambda: {}) - type: Literal["map"] = field(default="map") - - -@dataclass -class Const: - value: Any - outputs: dict[PortID, NodeIndex] = field(default_factory=lambda: {}) - inputs: dict[PortID, ValueRef] = field(default_factory=lambda: {}) - type: Literal["const"] = field(default="const") - - -@dataclass -class IfElse: - pred: ValueRef - if_true: ValueRef - if_false: ValueRef - outputs: dict[PortID, NodeIndex] = field(default_factory=lambda: {}) - inputs: dict[PortID, ValueRef] = field(default_factory=lambda: {}) - type: Literal["ifelse"] = field(default="ifelse") - - -@dataclass -class EagerIfElse: - pred: ValueRef - if_true: ValueRef - if_false: ValueRef - outputs: dict[PortID, NodeIndex] = field(default_factory=lambda: {}) - inputs: dict[PortID, ValueRef] = field(default_factory=lambda: {}) - - type: Literal["eifelse"] = field(default="eifelse") - - -@dataclass -class Input: - name: str - outputs: dict[PortID, NodeIndex] = field(default_factory=lambda: {}) - inputs: dict[PortID, ValueRef] = field(default_factory=lambda: {}) - type: Literal["input"] = field(default="input") - - -@dataclass -class Output: - inputs: dict[PortID, ValueRef] - outputs: dict[PortID, NodeIndex] = field(default_factory=lambda: {}) - type: Literal["output"] = field(default="output") - - -NodeDef = Func | Eval | Loop | Map | Const | IfElse | EagerIfElse | Input | Output -NodeDefModel = RootModel[NodeDef] - - -class GraphData(BaseModel): - nodes: list[NodeDef] = [] - fixed_inputs: dict[PortID, OutputLoc] = {} - graph_inputs: set[PortID] = set() - graph_output_idx: NodeIndex | None = None - named_nodes: dict[str, NodeIndex] = {} - - def input(self, name: str) -> ValueRef: - return self.add(Input(name))(name) - - def const(self, value: PType) -> ValueRef: - return self.add(Const(value))("value") - - def func( - self, function_name: str, inputs: dict[PortID, ValueRef] - ) -> Callable[[PortID], ValueRef]: - return self.add(Func(function_name, inputs)) - - def eval( - self, graph: ValueRef, inputs: dict[PortID, ValueRef] - ) -> Callable[[PortID], ValueRef]: - return self.add(Eval(graph, inputs)) - - def loop( - self, - body: ValueRef, - inputs: dict[PortID, ValueRef], - continue_port: PortID, - name: str | None = None, - ) -> Callable[[PortID], ValueRef]: - return self.add(Loop(body, inputs, continue_port, name=name)) - - def map( - self, - body: ValueRef, - inputs: dict[PortID, ValueRef], - ) -> Callable[[PortID], ValueRef]: - return self.add(Map(body, inputs)) - - def if_else(self, pred: ValueRef, if_true: ValueRef, if_false: ValueRef): - return self.add(IfElse(pred, if_true, if_false)) - - def eager_if_else(self, pred: ValueRef, if_true: ValueRef, if_false: ValueRef): - return self.add(EagerIfElse(pred, if_true, if_false)) - - def output(self, inputs: dict[PortID, ValueRef]) -> None: - self.add(Output(inputs)) - - def add(self, node: NodeDef) -> Callable[[PortID], ValueRef]: - idx = len(self.nodes) - self.nodes.append(node) - match node.type: - case "output": - if self.graph_output_idx is not None: - raise TierkreisError( - f"Graph already has output at index {self.graph_output_idx}" - ) - - self.graph_output_idx = idx - case "ifelse" | "eifelse": - self.nodes[node.pred[0]].outputs[node.pred[1]] = idx - self.nodes[node.if_true[0]].outputs[node.if_true[1]] = idx - self.nodes[node.if_false[0]].outputs[node.if_false[1]] = idx - case "input": - self.graph_inputs.add(node.name) - case "const" | "eval" | "function" | "map": - pass - case "loop": - if node.name is not None: - self.named_nodes[node.name] = idx - case _: - assert_never(node) - - for i, port in node.inputs.values(): - self.nodes[i].outputs[port] = idx - - return lambda k: (idx, k) - - def output_idx(self) -> NodeIndex: - idx = self.graph_output_idx - if idx is None: - raise TierkreisError("Graph has no output index.") - - node = self.nodes[idx] - if node.type != "output": - raise TierkreisError(f"Expected output node at {idx} found {node}") - - return idx - - def remaining_inputs(self, provided_inputs: set[PortID]) -> set[PortID]: - fixed_inputs = set(self.fixed_inputs.keys()) - if fixed_inputs & provided_inputs: - raise TierkreisError( - f"Fixed inputs {fixed_inputs}" - f" should not intersect provided inputs {provided_inputs}." - ) - - actual_inputs = fixed_inputs.union(provided_inputs) - return self.graph_inputs - actual_inputs - - -def graph_node_from_loc( - node_location: Loc, - graph: GraphData, -) -> tuple[NodeDef, GraphData]: - """Assumes the first part of a loc can be found in current graph""" - if len(graph.nodes) == 0: - raise TierkreisError("Cannot convert location to node. Reason: Empty Graph") - if node_location == "-": - return Eval((-1, "body"), {}), graph - - step, remaining_location = node_location.pop_first() - if isinstance(step, str): - raise TierkreisError("Cannot convert location: Reason: Malformed Loc") - (_, node_id) = step - if node_id == -1: - return Eval((-1, "body"), {}), graph - node = graph.nodes[node_id] - if remaining_location == Loc(): - return node, graph - match node.type: - case "eval": - graph = _unwrap_graph(graph.nodes[node.graph[0]], node.type) - node, graph = graph_node_from_loc(remaining_location, graph) - case "loop" | "map": - graph = _unwrap_graph(graph.nodes[node.body[0]], node.type) - _, remaining_location = remaining_location.pop_first() # Remove the M0/L0 - if len(remaining_location.steps()) < 2: - return Eval((-1, "body"), node.inputs, node.outputs), graph - - node, graph = graph_node_from_loc(remaining_location, graph) - case "const" | "function" | "input" | "output" | "ifelse" | "eifelse": - pass - case _: - assert_never(node) - - return node, graph - - -def _unwrap_graph(node: NodeDef, node_type: str) -> GraphData: - """Safely unwraps a const nodes GraphData.""" - if not isinstance(node, Const): - raise TierkreisError( - f"Cannot convert location to node. Reason: {node_type} does not wrap const" - ) - match node.value: - case GraphData() as graph: - return graph - case bytes() as thunk: - return GraphData(**ptype_from_bytes(thunk, dict)) - case dict() as data: - return GraphData(**data) - - case _: - raise TierkreisError( - "Cannot convert location to node. Reason: const value is not a graph" - ) diff --git a/tierkreis/tierkreis/controller/data/location.py b/tierkreis/tierkreis/controller/data/location.py index 1e5bbd8b3..1cc4df7e4 100644 --- a/tierkreis/tierkreis/controller/data/location.py +++ b/tierkreis/tierkreis/controller/data/location.py @@ -1,14 +1,9 @@ from logging import getLogger from pathlib import Path -from typing import Any, Literal, Optional +from typing import Optional -from pydantic import BaseModel, GetCoreSchemaHandler -from pydantic_core import CoreSchema, core_schema -from tierkreis.controller.data.core import PortID -from typing_extensions import assert_never - -from tierkreis.controller.data.core import NodeIndex -from tierkreis.exceptions import TierkreisError +from pydantic import BaseModel +from tierkreis_core import Loc, PortID logger = getLogger(__name__) @@ -23,112 +18,4 @@ class WorkerCallArgs(BaseModel): logs_path: Optional[Path] -NodeStep = Literal["-"] | tuple[Literal["N", "L", "M"], NodeIndex] - - -class Loc(str): - def __new__(cls, k: str = "-") -> "Loc": - return super(Loc, cls).__new__(cls, k) - - def N(self, idx: int) -> "Loc": - return Loc(str(self) + f".N{idx}") - - def L(self, idx: int) -> "Loc": - return Loc(str(self) + f".L{idx}") - - def M(self, idx: int) -> "Loc": - return Loc(str(self) + f".M{idx}") - - @staticmethod - def from_steps(steps: list[NodeStep]) -> "Loc": - loc = "" - for step in steps.copy(): - match step: - case "-": - loc += "-" - case (node_type, idx): - loc += f".{node_type}{idx}" - return Loc(loc) - - def parent(self) -> "Loc | None": - steps = self.steps() - if not steps: - return None - - last_step = steps.pop() - match last_step: - case "-": - return Loc.from_steps([]) - case ("L", 0): - return Loc.from_steps(steps) - case ("L", idx): - return Loc.from_steps(steps).L(idx - 1) - case ("N", idx) | ("M", idx): - return Loc.from_steps(steps) - case _: - assert_never(last_step) - - def steps(self) -> list[NodeStep]: - if self == "": - return [] - - steps: list[NodeStep] = [] - for step_str in self.split("."): - match step_str[0], step_str[1:]: - case ("-", _): - steps.append("-") - case ("N", idx_str): - steps.append(("N", int(idx_str))) - case ("L", idx_str): - steps.append(("L", int(idx_str))) - case ("M", idx_str): - steps.append(("M", int(idx_str))) - case _: - raise TierkreisError(f"Invalid Loc: {self}") - - return steps - - @classmethod - def __get_pydantic_core_schema__( - cls, source_type: Any, handler: GetCoreSchemaHandler - ) -> CoreSchema: - return core_schema.no_info_after_validator_function(cls, handler(str)) - - def pop_first(self) -> tuple[NodeStep, "Loc"]: - if self == "-": - return "-", Loc("") - steps = self.steps() - if len(steps) < 2: - raise TierkreisError("Malformed Loc") - first = steps.pop(1) - if first == "-": - raise TierkreisError("Malformed Loc") - return first, Loc.from_steps(steps) - - def pop_last(self) -> tuple[NodeStep, "Loc"]: - if self == "-": - return "-", Loc("") - steps = self.steps() - if len(steps) < 2: - raise TierkreisError("Malformed Loc") - last = steps.pop(-1) - if last == "-": - raise TierkreisError("Malformed Loc") - return last, Loc.from_steps(steps) - - def peek(self) -> NodeStep: - return self.steps()[-1] - - def peek_index(self) -> int: - step = self.steps()[-1] - - if isinstance(step, str): - return 0 - return step[1] - - def partial_locs(self) -> list["Loc"]: - steps = self.steps() - return [Loc.from_steps(steps[: i + 1]) for i in range(len(steps))] - - OutputLoc = tuple[Loc, PortID] diff --git a/tierkreis/tierkreis/controller/data/models.py b/tierkreis/tierkreis/controller/data/models.py index 50877a13a..d64997e42 100644 --- a/tierkreis/tierkreis/controller/data/models.py +++ b/tierkreis/tierkreis/controller/data/models.py @@ -15,12 +15,10 @@ ) from typing_extensions import TypeIs from tierkreis.controller.data.core import ( - NodeIndex, - PortID, RestrictedNamedTuple, - ValueRef, ) from tierkreis.controller.data.types import PType, generics_in_ptype +from tierkreis_core import ValueRef, NodeIndex, PortID TKR_PORTMAPPING_FLAG = "__tkr_portmapping__" @@ -49,7 +47,7 @@ class TKR[T: PModel]: port_id: PortID def value_ref(self) -> ValueRef: - return (self.node_index, self.port_id) + return ValueRef(self.node_index, self.port_id) @runtime_checkable @@ -108,9 +106,13 @@ def annotations_from_pmodel(pmodel: type) -> dict[PortID, Any]: def dict_from_tmodel(tmodel: TModel) -> dict[PortID, ValueRef]: if isinstance(tmodel, TNamedModel): - return {k: (v.node_index, v.port_id) for k, v in tmodel._asdict().items() if v} + return { + k: ValueRef(v.node_index, v.port_id) + for k, v in tmodel._asdict().items() + if v is not None + } - return {"value": (tmodel.node_index, tmodel.port_id)} + return {"value": ValueRef(tmodel.node_index, tmodel.port_id)} def model_fields(model: type[PModel] | type[TModel]) -> list[str]: @@ -129,13 +131,13 @@ def init_tmodel[T: TModel](tmodel: type[T], refs: list[ValueRef]) -> T: model = tmodel if not is_tnamedmodel(o) else o args: list[TKR] = [] for ref in refs: - key = ref[1].replace("-*", "") + key = ref.port_id.replace("-*", "") param = model.__annotations__[key] if get_origin(param) == Union: param = next(x for x in get_args(param) if x) - args.append(param(ref[0], ref[1])) + args.append(param(ref.node_index, ref.port_id)) return cast(T, model(*args)) - return tmodel(refs[0][0], refs[0][1]) + return tmodel(refs[0].node_index, refs[0].port_id) def generics_in_pmodel(pmodel: type[PModel]) -> set[str]: diff --git a/tierkreis/tierkreis/controller/start.py b/tierkreis/tierkreis/controller/start.py index 02da7ac0f..4285d0bda 100644 --- a/tierkreis/tierkreis/controller/start.py +++ b/tierkreis/tierkreis/controller/start.py @@ -5,14 +5,19 @@ import subprocess import sys -from tierkreis.controller.data.core import PortID from tierkreis.controller.data.types import bytes_from_ptype, ptype_from_bytes from tierkreis.controller.executor.in_memory_executor import InMemoryExecutor from tierkreis.controller.storage.adjacency import outputs_iter -from typing_extensions import assert_never - from tierkreis.consts import PACKAGE_PATH -from tierkreis.controller.data.graph import Eval, GraphData, NodeDef +from tierkreis_core import ( + PortID, + ExteriorRef, + GraphData, + NodeDef, + new_eval_root, + NodeDescription, + NodeIndex, +) from tierkreis.controller.data.location import Loc, OutputLoc from tierkreis.controller.executor.protocol import ControllerExecutor from tierkreis.controller.storage.protocol import ControllerStorage @@ -27,7 +32,7 @@ class NodeRunData: node_location: Loc node: NodeDef - output_list: list[PortID] + outputs: dict[PortID, NodeIndex] def start_nodes( @@ -72,104 +77,112 @@ def start( ) -> None: node_location = node_run_data.node_location node = node_run_data.node - output_list = node_run_data.output_list + node_outputs = node_run_data.outputs - storage.write_node_def(node_location, node) + storage.write_node_description(node_location, NodeDescription(node, node_outputs)) parent = node_location.parent() if parent is None: - raise TierkreisError(f"{node.type} node must have parent Loc.") + raise TierkreisError(f"{type(node)} node must have parent Loc.") + + ins = { + k: (parent.extend_from_ref(ref), ref.port_id) for k, ref in node.inputs.items() + } + + logger.debug(f"start {node_location} {node} {ins}") + match node: + case NodeDef.Func(): + name = node.name + launcher_name = ".".join(name.split(".")[:-1]) + name = name.split(".")[-1] + call_args_path = storage.write_worker_call_args( + node_location, name, ins, list(node_outputs) + ) + logger.debug(f"Executing {(str(node_location), name, ins, node_outputs)}") + + if isinstance(storage, ControllerInMemoryStorage) and isinstance( + executor, InMemoryExecutor + ): + executor.run(launcher_name, call_args_path) + elif launcher_name == "builtins": + run_builtin(call_args_path, storage.logs_path) + else: + executor.run(launcher_name, call_args_path) + + case NodeDef.Input(): + input_loc = parent.exterior() + storage.link_outputs(node_location, node.name, input_loc, node.name) + storage.mark_node_finished(node_location) - ins = {k: (parent.N(idx), p) for k, (idx, p) in node.inputs.items()} + case NodeDef.Output(): + storage.mark_node_finished(node_location) - logger.debug(f"start {node_location} {node} {ins} {output_list}") - if node.type == "function": - name = node.function_name - launcher_name = ".".join(name.split(".")[:-1]) - name = name.split(".")[-1] - call_args_path = storage.write_worker_call_args( - node_location, name, ins, output_list - ) - logger.debug(f"Executing {(str(node_location), name, ins, output_list)}") - - if isinstance(storage, ControllerInMemoryStorage) and isinstance( - executor, InMemoryExecutor - ): - executor.run(launcher_name, call_args_path) - elif launcher_name == "builtins": - run_builtin(call_args_path, storage.logs_path) - else: - executor.run(launcher_name, call_args_path) - - elif node.type == "input": - input_loc = parent.N(-1) - storage.link_outputs(node_location, node.name, input_loc, node.name) - storage.mark_node_finished(node_location) - - elif node.type == "output": - storage.mark_node_finished(node_location) - - pipe_inputs_to_output_location(storage, parent, ins) - storage.mark_node_finished(parent) - - elif node.type == "const": - bs = bytes_from_ptype(node.value) - storage.write_output(node_location, Labels.VALUE, bs) - storage.mark_node_finished(node_location) - - elif node.type == "eval": - message = storage.read_output(parent.N(node.graph[0]), node.graph[1]) - g = ptype_from_bytes(message, GraphData) - ins["body"] = (parent.N(node.graph[0]), node.graph[1]) - ins.update(g.fixed_inputs) - - pipe_inputs_to_output_location(storage, node_location.N(-1), ins) - - elif node.type == "loop": - ins["body"] = (parent.N(node.body[0]), node.body[1]) - pipe_inputs_to_output_location(storage, node_location.N(-1), ins) - if ( - node.name is not None - ): # should we do this only in debug mode? -> need to think through how this would work - storage.write_debug_data(node.name, node_location) - start( - storage, - executor, - NodeRunData( - node_location.L(0), - Eval((-1, "body"), {k: (-1, k) for k, _ in ins.items()}, node.outputs), - output_list, - ), - ) + pipe_inputs_to_output_location(storage, parent, ins) + storage.mark_node_finished(parent) - elif node.type == "map": - first_ref = next(x for x in ins.values() if x[1] == "*") - map_eles = outputs_iter(storage, first_ref[0]) - if not map_eles: + case NodeDef.Const(): + bs = bytes_from_ptype(node.value) + storage.write_output(node_location, Labels.VALUE, bs) storage.mark_node_finished(node_location) - for idx, p in map_eles: - eval_inputs: dict[PortID, tuple[Loc, PortID]] = {} - eval_inputs["body"] = (parent.N(node.body[0]), node.body[1]) - for k, (i, port) in ins.items(): - if port == "*": - eval_inputs[k] = (i, p) - else: - eval_inputs[k] = (i, port) - pipe_inputs_to_output_location( - storage, node_location.M(idx).N(-1), eval_inputs - ) - # Necessary in the node visualization - storage.write_node_def( - node_location.M(idx), Eval((-1, "body"), node.inputs, node.outputs) - ) - elif node.type == "ifelse": - pass + case NodeDef.Eval(): + body_loc = parent.extend_from_ref(node.body) + message = storage.read_output(body_loc, node.body.port_id) + g = ptype_from_bytes(message, GraphData) + ins["body"] = (body_loc, node.body.port_id) + ins.update(g.fixed_inputs) + + pipe_inputs_to_output_location(storage, node_location.exterior(), ins) + + case NodeDef.Loop(): + if ( + node.name is not None + ): # should we do this only in debug mode? -> need to think through how this would work + storage.write_debug_data(node.name, node_location) + ins["body"] = (parent.extend_from_ref(node.body), node.body.port_id) + pipe_inputs_to_output_location(storage, node_location.exterior(), ins) + start( + storage, + executor, + NodeRunData( + node_location.L(0), + new_eval_root({k: ExteriorRef(k) for k in ins.keys()}), + node_outputs, + ), + ) - elif node.type == "eifelse": - pass - else: - assert_never(node) + case NodeDef.Map(): + first_ref = next(x for x in ins.values() if x[1] == "*") + map_eles = outputs_iter(storage, first_ref[0]) + if not map_eles: + storage.mark_node_finished(node_location) + for idx, p in map_eles: + eval_inputs: dict[PortID, tuple[Loc, PortID]] = {} + eval_inputs["body"] = ( + parent.extend_from_ref(node.body), + node.body.port_id, + ) + for k, (i, port) in ins.items(): + if port == "*": + eval_inputs[k] = (i, p) + else: + eval_inputs[k] = (i, port) + pipe_inputs_to_output_location( + storage, node_location.M(idx).exterior(), eval_inputs + ) + # Necessary in the node visualization + storage.write_node_description( + node_location.M(idx), + NodeDescription(new_eval_root(node.inputs), node_outputs), + ) + + case NodeDef.IfElse(): + pass + + case NodeDef.EagerIfElse(): + pass + case _: + raise ValueError(f"Unhandled NodeDef of type: {type(node)}") def pipe_inputs_to_output_location( diff --git a/tierkreis/tierkreis/controller/storage/adjacency.py b/tierkreis/tierkreis/controller/storage/adjacency.py index 2c5e1e3a7..21f7014f0 100644 --- a/tierkreis/tierkreis/controller/storage/adjacency.py +++ b/tierkreis/tierkreis/controller/storage/adjacency.py @@ -1,48 +1,21 @@ import logging -from typing import assert_never -from tierkreis.controller.data.core import PortID, ValueRef -from tierkreis.controller.data.graph import ( - NodeDef, -) from tierkreis.controller.data.location import Loc from tierkreis.controller.storage.protocol import ControllerStorage +from tierkreis_core import ValueRef, NodeDef, PortID logger = logging.getLogger(__name__) -def in_edges(node: NodeDef) -> dict[PortID, ValueRef]: - parents = {k: v for k, v in node.inputs.items()} - - match node.type: - case "eval": - parents["body"] = node.graph - case "loop": - parents["body"] = node.body - case "map": - parents["body"] = node.body - case "ifelse": - parents["pred"] = node.pred - case "eifelse": - parents["pred"] = node.pred - parents["body_true"] = node.if_true - parents["body_false"] = node.if_false - case "const" | "function" | "input" | "output": - pass - case _: - assert_never(node) - - return parents - - def unfinished_inputs( storage: ControllerStorage, loc: Loc, node: NodeDef ) -> list[ValueRef]: - ins = in_edges(node).values() - ins = [x for x in ins if x[0] >= 0] # inputs at -1 already finished - return [x for x in ins if not storage.is_node_finished(loc.N(x[0]))] + ins = node.in_edges.values() + ins = [x for x in ins if isinstance(x, ValueRef)] # Only look an Values on Edges + return [x for x in ins if not storage.is_node_finished(loc.N(x.node_index))] def outputs_iter(storage: ControllerStorage, loc: Loc) -> list[tuple[int, PortID]]: + """Retrieve the indexes for a virtual map node by parsing output port names.""" eles = storage.read_output_ports(loc) return [(int(x.split("-")[-1]), x) for x in eles] diff --git a/tierkreis/tierkreis/controller/storage/filestorage.py b/tierkreis/tierkreis/controller/storage/filestorage.py index 41885cbc3..9300e675e 100644 --- a/tierkreis/tierkreis/controller/storage/filestorage.py +++ b/tierkreis/tierkreis/controller/storage/filestorage.py @@ -5,8 +5,8 @@ from uuid import UUID from tierkreis.controller.storage.protocol import ( - StorageEntryMetadata, ControllerStorage, + StorageEntryMetadata, ) diff --git a/tierkreis/tierkreis/controller/storage/graphdata.py b/tierkreis/tierkreis/controller/storage/graphdata.py index 168b1484c..d154fa99b 100644 --- a/tierkreis/tierkreis/controller/storage/graphdata.py +++ b/tierkreis/tierkreis/controller/storage/graphdata.py @@ -4,24 +4,31 @@ from pydantic import BaseModel, Field -from tierkreis.controller.data.core import PortID -from tierkreis.controller.data.graph import ( - Eval, - GraphData, - NodeDef, - graph_node_from_loc, +from tierkreis.controller.data.location import ( + Loc, + OutputLoc, + WorkerCallArgs, ) -from tierkreis.controller.data.location import Loc, OutputLoc, WorkerCallArgs from tierkreis.controller.storage.protocol import ( StorageEntryMetadata, ControllerStorage, ) from tierkreis.exceptions import TierkreisError +from tierkreis_core import ( + PortID, + NodeDef, + GraphData, + NodeDescription, + NodeStep, + new_eval_root, +) class NodeData(BaseModel): """Internal storage class to store all necessary node information.""" + model_config = {"arbitrary_types_allowed": True} + definition: NodeDef | None = None call_args: WorkerCallArgs | None = None is_done: bool = False @@ -73,17 +80,18 @@ def touch(self, path: Path, is_dir: bool = False) -> None: def write(self, path: Path, value: bytes) -> None: raise NotImplementedError("GraphDataStorage is read only storage.") - def write_node_def(self, node_location: Loc, node: NodeDef) -> None: + def write_node_description(self, node_location: Loc, node: NodeDescription) -> None: raise NotImplementedError("GraphDataStorage is read only storage.") - def read_node_def(self, node_location: Loc) -> NodeDef: + def read_node_description(self, node_location: Loc) -> NodeDescription: try: - if node_location.pop_last()[0][0] in ["M", "L"]: - return Eval((-1, "body"), {}) + (last_step, _) = node_location.pop_last() + if isinstance(last_step, (NodeStep.Map, NodeStep.Loop)): + return NodeDescription(new_eval_root({})) except (TierkreisError, TypeError): - return Eval((-1, "body"), {}) - node, _ = graph_node_from_loc(node_location, self.graph) - return node + return NodeDescription(new_eval_root({})) + desc = self.graph.query_node_description(node_location) + return desc def write_worker_call_args( self, @@ -129,11 +137,13 @@ def write_output( raise NotImplementedError("GraphDataStorage is read only storage.") def read_output(self, node_location: Loc, output_name: PortID) -> bytes: - node, graph = graph_node_from_loc(node_location, self.graph) - if -1 == node_location.peek_index() and output_name == "body": - return graph.model_dump_json().encode() + desc = self.graph.query_node_description(node_location) + if node_location.last_step_exterior() and output_name == "body": + if desc.outer_graph is None: + raise ValueError("Empty outer_graph for node with a `body` port") + return desc.outer_graph.model_dump_json().encode() - outputs = _build_node_outputs(node) + outputs = _build_node_outputs(desc) if output_name in outputs: if output := outputs[output_name]: return output @@ -141,8 +151,8 @@ def read_output(self, node_location: Loc, output_name: PortID) -> bytes: raise TierkreisError(f"No output named {output_name} in node {node_location}") def read_output_ports(self, node_location: Loc) -> list[PortID]: - node, _ = graph_node_from_loc(node_location, self.graph) - outputs = _build_node_outputs(node) + desc = self.graph.query_node_description(node_location) + outputs = _build_node_outputs(desc) return list(filter(lambda k: k != "*", outputs.keys())) def is_node_started(self, node_location: Loc) -> bool: @@ -161,8 +171,8 @@ def read_finished_time(self, node_location: Loc) -> str | None: return None -def _build_node_outputs(node: NodeDef) -> dict[PortID, None | bytes]: - outputs: dict[PortID, None | bytes] = {val: None for val in node.outputs} +def _build_node_outputs(desc: NodeDescription) -> dict[PortID, None | bytes]: + outputs: dict[PortID, None | bytes] = {val: None for val in desc.outputs} if "*" in outputs: outputs["0"] = None return outputs diff --git a/tierkreis/tierkreis/controller/storage/protocol.py b/tierkreis/tierkreis/controller/storage/protocol.py index 6f29d22c4..d2daae708 100644 --- a/tierkreis/tierkreis/controller/storage/protocol.py +++ b/tierkreis/tierkreis/controller/storage/protocol.py @@ -4,12 +4,11 @@ import json import logging from pathlib import Path -from typing import Any, assert_never +from typing import Any from uuid import UUID -from tierkreis.controller.data.graph import NodeDef, NodeDefModel from tierkreis.controller.data.location import Loc, OutputLoc, WorkerCallArgs -from tierkreis.controller.data.core import PortID from tierkreis.exceptions import TierkreisError +from tierkreis_core import NodeDef, NodeDescription, PortID, NodeStep logger = logging.getLogger(__name__) @@ -120,13 +119,13 @@ def _error_logs_path(self, node_location: Loc) -> Path: def clean_graph_files(self) -> None: self.delete(self.workflow_dir) - def write_node_def(self, node_location: Loc, node: NodeDef): - bs = NodeDefModel(root=node).model_dump_json().encode() + def write_node_description(self, node_location: Loc, node: NodeDescription): + bs = node.model_dump_json().encode() self.write(self._nodedef_path(node_location), bs) - def read_node_def(self, node_location: Loc) -> NodeDef: + def read_node_description(self, node_location: Loc) -> NodeDescription: bs = self.read(self._nodedef_path(node_location)) - return NodeDefModel(**json.loads(bs)).root + return NodeDescription.model_load_json(bs.decode()) def write_worker_call_args( self, @@ -259,8 +258,8 @@ def read_finished_time(self, node_location: Loc) -> str | None: return datetime.fromtimestamp(since_epoch).isoformat() def read_loop_trace(self, node_location: Loc, output_name: PortID) -> list[bytes]: - definition = self.read_node_def(node_location) - if definition.type != "loop": + description = self.read_node_description(node_location) + if not isinstance(description.definition, NodeDef.Loop): raise TierkreisError("Can only read traces from loop nodes.") result = [] @@ -277,7 +276,7 @@ def loc_from_node_name(self, node_name: str) -> Loc | None: def write_debug_data(self, name: str, loc: Loc) -> None: self.mkdir(self.debug_path) - data = StorageDebugData(loop_loc=loc) + data = StorageDebugData(loop_loc=str(loc)) self.write(self.debug_path / name, json.dumps(asdict(data)).encode()) def read_debug_data(self, name: str) -> dict[str, Any]: @@ -291,23 +290,27 @@ def dependents(self, loc: Loc) -> set[Loc]: descs: set[Loc] = set() step, parent = loc.pop_last() match step: - case "-": + case NodeStep.Root(): pass - case ("N", _): - nodedef = self.read_node_def(loc) - if nodedef.type == "output": + case NodeStep.Node(): + nodedef = self.read_node_description(loc) + if isinstance(nodedef.definition, NodeDef.Output): descs.update(self.dependents(parent)) for output in nodedef.outputs.values(): descs.add(parent.N(output)) descs.update(self.dependents(parent.N(output))) - case ("M", _): + case NodeStep.Map(): descs.update(self.dependents(parent)) - case ("L", idx): + case NodeStep.Loop(): latest_idx = self.latest_loop_iteration(parent).peek_index() - [descs.add(parent.L(i)) for i in range(idx + 1, latest_idx + 1)] + assert latest_idx is not None # Should be impossible + [ + descs.add(parent.L(i)) + for i in range(step.loop_index + 1, latest_idx + 1) + ] descs.update(self.dependents(parent)) case _: - assert_never(step) + raise ValueError(f"Unhandled NodeStep of type: {type(step)}") return descs @@ -319,8 +322,8 @@ def restart_task(self, loc: Loc) -> list[Loc]: Returns the invalidated nodes.""" - nodedef = self.read_node_def(loc) - if nodedef.type != "function": + nodedef = self.read_node_description(loc) + if not isinstance(nodedef.definition, NodeDef.Func): raise TierkreisError("Can only restart task/function nodes.") # Remove fully invalidated nodes. diff --git a/tierkreis/tierkreis/controller/storage/walk.py b/tierkreis/tierkreis/controller/storage/walk.py index 2f4bbbbfb..1a71fd29a 100644 --- a/tierkreis/tierkreis/controller/storage/walk.py +++ b/tierkreis/tierkreis/controller/storage/walk.py @@ -1,23 +1,22 @@ from dataclasses import dataclass, field from logging import getLogger -from typing import assert_never from tierkreis.controller.consts import BODY_PORT -from tierkreis.controller.data.core import NodeIndex -from tierkreis.controller.data.graph import ( - EagerIfElse, - Eval, - GraphData, - Loop, - Map, - NodeDef, -) from tierkreis.controller.data.location import Loc from tierkreis.controller.data.types import ptype_from_bytes from tierkreis.controller.start import NodeRunData from tierkreis.controller.storage.adjacency import outputs_iter, unfinished_inputs from tierkreis.controller.storage.protocol import ControllerStorage from tierkreis.labels import Labels +from tierkreis_core import ( + ExteriorRef, + GraphData, + NodeDef, + NodeIndex, + ValueRef, + PortID, + new_eval_root, +) logger = getLogger(__name__) @@ -42,7 +41,7 @@ def unfinished_results( graph: GraphData, ) -> int: unfinished = unfinished_inputs(storage, parent, node) - [result.extend(walk_node(storage, parent, x[0], graph)) for x in unfinished] + [result.extend(walk_node(storage, parent, x.node_index, graph)) for x in unfinished] return len(unfinished) @@ -56,8 +55,12 @@ def walk_node( logger.error(f"\n\n{storage.read_errors(loc)}\n\n") return WalkResult([], [], [loc]) - node = graph.nodes[idx] - node_run_data = NodeRunData(loc, node, list(node.outputs)) + node = graph.get_nodedef(idx) + graph_outputs = graph.outputs(idx) + if graph_outputs is None: + raise ValueError("Cannot walk a graph with no outputs.") + + node_run_data = NodeRunData(loc, node, graph_outputs) result = WalkResult([], []) if unfinished_results(result, storage, parent, node, graph): @@ -66,49 +69,58 @@ def walk_node( if not storage.is_node_started(loc): return WalkResult([node_run_data], []) - match node.type: - case "eval": - message = storage.read_output(parent.N(node.graph[0]), node.graph[1]) + match node: + case NodeDef.Eval(): + message = storage.read_output( + parent.extend_from_ref(node.body), node.body.port_id + ) g = ptype_from_bytes(message, GraphData) - return walk_node(storage, loc, g.output_idx(), g) - case "output": + output_idx = g.output_idx() + if output_idx is None: + raise ValueError("Cannot walk a graph with no Output node.") + + return walk_node(storage, loc, output_idx, g) + + case NodeDef.Output(): return WalkResult([node_run_data], []) - case "const": + case NodeDef.Const(): return WalkResult([node_run_data], []) - case "loop": + case NodeDef.Loop(): return walk_loop(storage, parent, idx, node) - case "map": + case NodeDef.Map(): return walk_map(storage, parent, idx, node) - case "ifelse": - pred = storage.read_output(parent.N(node.pred[0]), node.pred[1]) + case NodeDef.IfElse(): + pred = storage.read_output( + parent.extend_from_ref(node.pred), node.pred.port_id + ) next_node = node.if_true if pred == b"true" else node.if_false - next_loc = parent.N(next_node[0]) + next_loc = parent.extend_from_ref(next_node) if storage.is_node_finished(next_loc): - storage.link_outputs(loc, Labels.VALUE, next_loc, next_node[1]) + storage.link_outputs(loc, Labels.VALUE, next_loc, next_node.port_id) storage.mark_node_finished(loc) return WalkResult([], []) else: - return walk_node(storage, parent, next_node[0], graph) + return walk_node(storage, parent, next_node.node_index, graph) - case "eifelse": - return walk_eifelse(storage, parent, idx, node) + case NodeDef.EagerIfElse(): + return walk_eagerifelse(storage, parent, idx, node) - case "function": + case NodeDef.Func(): return WalkResult([], [loc]) - case "input": + case NodeDef.Input(): return WalkResult([], []) case _: - assert_never(node) + raise ValueError(f"Unhandled NodeDef of type: {type(node)}") def walk_loop( - storage: ControllerStorage, parent: Loc, idx: NodeIndex, loop: Loop + storage: ControllerStorage, parent: Loc, idx: NodeIndex, loop: NodeDef.Loop ) -> WalkResult: loc = parent.N(idx) if storage.is_node_finished(loc): @@ -116,52 +128,81 @@ def walk_loop( new_location = storage.latest_loop_iteration(loc) - message = storage.read_output(loc.N(-1), BODY_PORT) + message = storage.read_output(loc.exterior(), BODY_PORT) g = ptype_from_bytes(message, GraphData) - loop_outputs = g.nodes[g.output_idx()].inputs + output_idx = g.output_idx() + if output_idx is None: + raise ValueError("Cannot walk a graph with no Output node.") if not storage.is_node_finished(new_location): - return walk_node(storage, new_location, g.output_idx(), g) + return walk_node(storage, new_location, output_idx, g) + + # The outputs from the previous iteration + body_outputs = g.get_nodedef(output_idx).inputs + if body_outputs is None: + raise ValueError("Loop body has no outputs.") # Latest iteration is finished. Do we BREAK or CONTINUE? should_continue = ptype_from_bytes( storage.read_output(new_location, loop.continue_port), bool ) if should_continue is False: - for k in loop_outputs: + for k in body_outputs: storage.link_outputs(loc, k, new_location, k) storage.mark_node_finished(loc) return WalkResult([], []) - ins = {k: (-1, k) for k in loop.inputs.keys()} - ins.update(loop_outputs) + # Create new exterior refs for the inputs to the loop node + # + # This allows us to re-use the inputs to the loop node + # between iterations. + ins: dict[PortID, ValueRef | ExteriorRef] = { + k: ExteriorRef(k) for k in loop.inputs.keys() + } + # Update with the outputs of the subgraph. + ins.update(body_outputs) + + previous_index = new_location.peek_index() + if previous_index is None: + # TODO: This should be impossible + raise ValueError("Previous step is not a Loop step.") + + # The outputs from the previous iteration + graph_outputs = g.graph_outputs() + if graph_outputs is None: + raise ValueError("Loop body has no outputs.") + node_run_data = NodeRunData( - loc.L(new_location.peek_index() + 1), - Eval((-1, BODY_PORT), ins, loop.outputs), - list(loop_outputs.keys()), + loc.L(previous_index + 1), + new_eval_root(ins), + graph_outputs, ) return WalkResult([node_run_data], []) def walk_map( - storage: ControllerStorage, parent: Loc, idx: NodeIndex, map: Map + storage: ControllerStorage, parent: Loc, idx: NodeIndex, map: NodeDef.Map ) -> WalkResult: loc = parent.N(idx) result = WalkResult([], []) if storage.is_node_finished(loc): return result - first_ref = next(x for x in map.inputs.values() if x[1] == "*") - map_eles = outputs_iter(storage, parent.N(first_ref[0])) + first_ref = next(x for x in map.inputs.values() if x.port_id == "*") + map_eles = outputs_iter(storage, parent.extend_from_ref(first_ref)) unfinished = [i for i, _ in map_eles if not storage.is_node_finished(loc.M(i))] - message = storage.read_output(loc.M(0).N(-1), BODY_PORT) + message = storage.read_output(loc.M(0).exterior(), BODY_PORT) g = ptype_from_bytes(message, GraphData) - [result.extend(walk_node(storage, loc.M(p), g.output_idx(), g)) for p in unfinished] + output_idx = g.output_idx() + if output_idx is None: + raise ValueError("Cannot walk a graph with no Output node.") + + [result.extend(walk_node(storage, loc.M(p), output_idx, g)) for p in unfinished] if len(unfinished) > 0: return result - map_outputs = g.nodes[g.output_idx()].inputs + map_outputs = g.get_nodedef(output_idx).inputs for i, j in map_eles: for output in map_outputs.keys(): storage.link_outputs(loc, f"{output}-{j}", loc.M(i), output) @@ -170,17 +211,17 @@ def walk_map( return result -def walk_eifelse( +def walk_eagerifelse( storage: ControllerStorage, parent: Loc, idx: NodeIndex, - node: EagerIfElse, + node: NodeDef.EagerIfElse, ) -> WalkResult: loc = parent.N(idx) - pred = storage.read_output(parent.N(node.pred[0]), node.pred[1]) + pred = storage.read_output(parent.extend_from_ref(node.pred), node.pred.port_id) next_node = node.if_true if pred == b"true" else node.if_false - next_loc = parent.N(next_node[0]) - storage.link_outputs(loc, Labels.VALUE, next_loc, next_node[1]) + next_loc = parent.extend_from_ref(next_node) + storage.link_outputs(loc, Labels.VALUE, next_loc, next_node.port_id) storage.mark_node_finished(loc) return WalkResult([], []) diff --git a/tierkreis/tierkreis/graphs/fold.py b/tierkreis/tierkreis/graphs/fold.py index 274cabb98..199487f61 100644 --- a/tierkreis/tierkreis/graphs/fold.py +++ b/tierkreis/tierkreis/graphs/fold.py @@ -1,9 +1,9 @@ from typing import Generic, NamedTuple, TypeVar from tierkreis.builder import GraphBuilder, TypedGraphRef from tierkreis.builtins.stubs import head, igt, tkr_len -from tierkreis.controller.data.graph import GraphData from tierkreis.controller.data.models import TKR from tierkreis.controller.data.types import PType +from tierkreis_core import GraphData class FoldGraphOuterInputs[A: PType, B: PType](NamedTuple): diff --git a/tierkreis/tierkreis/storage.py b/tierkreis/tierkreis/storage.py index 1c70693e4..b0c20aea4 100644 --- a/tierkreis/tierkreis/storage.py +++ b/tierkreis/tierkreis/storage.py @@ -1,5 +1,4 @@ from tierkreis.builder import GraphBuilder -from tierkreis.controller.data.graph import GraphData from tierkreis.controller.data.location import Loc from tierkreis.controller.data.types import PType, ptype_from_bytes from tierkreis.controller.storage.protocol import ControllerStorage @@ -10,6 +9,7 @@ ControllerInMemoryStorage as InMemoryStorage, ) from tierkreis.exceptions import TierkreisError +from tierkreis_core import GraphData __all__ = ["FileStorage", "InMemoryStorage"] @@ -20,7 +20,11 @@ def read_outputs( if isinstance(g, GraphBuilder): g = g.get_data() - out_ports = list(g.nodes[g.output_idx()].inputs.keys()) + output_idx = g.output_idx() + if output_idx is None: + raise ValueError("Cannot read outputs of a graph with no Output node.") + + out_ports = list(g.get_nodedef(output_idx).inputs.keys()) if len(out_ports) == 1 and "value" in out_ports: return ptype_from_bytes(storage.read_output(Loc(), "value")) return {k: ptype_from_bytes(storage.read_output(Loc(), k)) for k in out_ports} diff --git a/tierkreis/tierkreis/worker/worker.py b/tierkreis/tierkreis/worker/worker.py index 6aee6454a..b9f4b3964 100644 --- a/tierkreis/tierkreis/worker/worker.py +++ b/tierkreis/tierkreis/worker/worker.py @@ -6,7 +6,7 @@ from types import TracebackType from typing import Callable, TypeVar -from tierkreis.controller.data.core import PortID +from tierkreis_core import PortID from tierkreis.controller.data.location import WorkerCallArgs from tierkreis.controller.data.models import ( PModel, diff --git a/tierkreis_core/.gitignore b/tierkreis_core/.gitignore new file mode 100644 index 000000000..01ab719f6 --- /dev/null +++ b/tierkreis_core/.gitignore @@ -0,0 +1,72 @@ +/target + +# Byte-compiled / optimized / DLL files +__pycache__/ +.pytest_cache/ +*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +.Python +.venv/ +env/ +# bin/ +build/ +develop-eggs/ +dist/ +eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +include/ +man/ +venv/ +*.egg-info/ +.installed.cfg +*.egg + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt +pip-selfcheck.json + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.cache +nosetests.xml +coverage.xml + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Rope +.ropeproject + +# Django stuff: +*.log +*.pot + +.DS_Store + +# Sphinx documentation +docs/_build/ + +# PyCharm +.idea/ + +# VSCode +.vscode/ + +# Pyenv +.python-version diff --git a/tierkreis_core/Cargo.lock b/tierkreis_core/Cargo.lock new file mode 100644 index 000000000..9a08e4621 --- /dev/null +++ b/tierkreis_core/Cargo.lock @@ -0,0 +1,375 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "goblin" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4db6758c546e6f81f265638c980e5e84dfbda80cfd8e89e02f83454c8e8124bd" +dependencies = [ + "log", + "plain", + "scroll", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "indexmap" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" +dependencies = [ + "equivalent", + "hashbrown", + "serde", + "serde_core", +] + +[[package]] +name = "indoc" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "libc" +version = "0.2.177" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" + +[[package]] +name = "log" +version = "0.4.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" + +[[package]] +name = "memchr" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + +[[package]] +name = "portable-atomic" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" + +[[package]] +name = "proc-macro2" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab53c047fcd1a1d2a8820fe84f05d6be69e9526be40cb03b73f86b6b03e6d87d" +dependencies = [ + "indexmap", + "indoc", + "libc", + "memoffset", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b455933107de8642b4487ed26d912c2d899dec6114884214a0b3bb3be9261ea6" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c85c9cbfaddf651b1221594209aed57e9e5cff63c4d11d1feead529b872a089" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-introspection" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6cd5679197d3cfdd7d9d241edf688ec13efd4fc302b35e8596f2831a0be5109" +dependencies = [ + "anyhow", + "goblin", + "serde", + "serde_json", + "unicode-ident", +] + +[[package]] +name = "pyo3-macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a5b10c9bf9888125d917fb4d2ca2d25c8df94c7ab5a52e13313a07e050a3b02" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03b51720d314836e53327f5871d4c0cfb4fb37cc2c4a11cc71907a86342c40f9" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn", +] + +[[package]] +name = "pythonize" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3a8f29db331e28c332c63496cfcbb822aca3d7320bc08b655d7fd0c29c50ede" +dependencies = [ + "pyo3", + "serde", +] + +[[package]] +name = "quote" +version = "1.0.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "scroll" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1257cd4248b4132760d6524d6dda4e053bc648c9070b960929bf50cfb1e7add" +dependencies = [ + "scroll_derive", +] + +[[package]] +name = "scroll_derive" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed76efe62313ab6610570951494bdaa81568026e0318eaa55f167de70eeea67d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.145" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", + "serde_core", +] + +[[package]] +name = "syn" +version = "2.0.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df7f62577c25e07834649fc3b39fafdc597c0a3527dc1c60129201ccfcbaa50c" + +[[package]] +name = "tierkreis_core" +version = "0.1.0" +dependencies = [ + "anyhow", + "indexmap", + "pyo3", + "pyo3-introspection", + "pythonize", + "regex", + "serde", + "serde_json", +] + +[[package]] +name = "unicode-ident" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" + +[[package]] +name = "unindent" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" diff --git a/tierkreis_core/Cargo.toml b/tierkreis_core/Cargo.toml new file mode 100644 index 000000000..26af6e448 --- /dev/null +++ b/tierkreis_core/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "tierkreis_core" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lib] +name = "tierkreis_core" +# There is also an rlib target to use for type stub generation. +crate-type = ["cdylib", "rlib"] +path = "rust/lib.rs" + +[dependencies] +anyhow = "1.0.98" +indexmap = { version = "2.12.0", features = ["serde"] } +pyo3 = { version = "0.27.0", features = ["indexmap", "abi3-py310", "experimental-inspect"] } +pythonize = "0.27.0" +serde = { version = "1.0.219", features = ["serde_derive"] } +serde_json = "1.0.141" + +# For stub generation +pyo3-introspection = { version = "0.27", optional = true } +regex = { version = "1.12", optional = true } + +[features] +stub-generation = ["pyo3-introspection", "regex"] + +[[bin]] +name = "tierkreis-core-stubs-gen" +required-features = ["stub-generation"] +path = "rust/bin/tierkreis-core-stubs-gen.rs" diff --git a/tierkreis_core/pyproject.toml b/tierkreis_core/pyproject.toml new file mode 100644 index 000000000..dd8e2de0a --- /dev/null +++ b/tierkreis_core/pyproject.toml @@ -0,0 +1,22 @@ +[build-system] +requires = ["maturin>=1.8,<2.0"] +build-backend = "maturin" + +[project] +name = "tierkreis_core" +requires-python = ">=3.8" +classifiers = [ + "Programming Language :: Rust", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] +dynamic = ["version"] + +[tool.maturin] +module-name = "tierkreis_core._tierkreis_core" +python-source = "python/" +manifest-path = "Cargo.toml" +features = ["pyo3/extension-module"] + +[tool.uv] +cache-keys = [{file = "pyproject.toml"}, {file = "Cargo.toml"}, {file = "rust/*.rs"}] diff --git a/tierkreis_core/python/tierkreis_core/__init__.py b/tierkreis_core/python/tierkreis_core/__init__.py new file mode 100644 index 000000000..f40abc83b --- /dev/null +++ b/tierkreis_core/python/tierkreis_core/__init__.py @@ -0,0 +1,23 @@ +from tierkreis_core._tierkreis_core.graph import ( # pyright: ignore[reportMissingModuleSource] + GraphData, + NodeDef, + new_eval_root, + NodeDescription, +) +from tierkreis_core._tierkreis_core.location import Loc, NodeStep # pyright: ignore[reportMissingModuleSource] +from tierkreis_core._tierkreis_core.identifiers import ValueRef, NodeIndex, ExteriorRef # pyright: ignore[reportMissingModuleSource] +from .aliases import PortID, Value + +__all__ = [ + "GraphData", + "Loc", + "PortID", + "Value", + "ValueRef", + "NodeIndex", + "ExteriorRef", + "NodeDef", + "new_eval_root", + "NodeDescription", + "NodeStep", +] diff --git a/tierkreis_core/python/tierkreis_core/_tierkreis_core/__init__.pyi b/tierkreis_core/python/tierkreis_core/_tierkreis_core/__init__.pyi new file mode 100644 index 000000000..b88c3a5f3 --- /dev/null +++ b/tierkreis_core/python/tierkreis_core/_tierkreis_core/__init__.pyi @@ -0,0 +1,3 @@ +import _typeshed + +def __getattr__(name: str) -> _typeshed.Incomplete: ... diff --git a/tierkreis_core/python/tierkreis_core/_tierkreis_core/graph.pyi b/tierkreis_core/python/tierkreis_core/_tierkreis_core/graph.pyi new file mode 100644 index 000000000..6c13d8058 --- /dev/null +++ b/tierkreis_core/python/tierkreis_core/_tierkreis_core/graph.pyi @@ -0,0 +1,158 @@ +import _typeshed +import tierkreis_core._tierkreis_core.graph as graph +import tierkreis_core._tierkreis_core.identifiers as identifiers +import tierkreis_core._tierkreis_core.location as location +import tierkreis_core.aliases +import tierkreis_core.nodes +import typing + +class CurriedNodeIndex: + def __call__(self, /, port: str) -> identifiers.ValueRef: ... + +class GraphData: + def __eq__(self, /, other: object) -> bool: ... + def __ne__(self, /, other: object) -> bool: ... + def __init__(self, /) -> None: ... + def const(self, /, value: tierkreis_core.aliases.Value) -> identifiers.ValueRef: ... + def eager_if_else( + self, + /, + pred: identifiers.ValueRef, + if_true: identifiers.ValueRef, + if_false: identifiers.ValueRef, + ) -> graph.CurriedNodeIndex: ... + def eval( + self, + /, + graph: identifiers.ExteriorRef | identifiers.ValueRef, + inputs: typing.Mapping[tierkreis_core.aliases.PortID, identifiers.ValueRef], + ) -> graph.CurriedNodeIndex: ... + @property + def fixed_inputs(self, /) -> typing.Any: ... + @classmethod + def from_dict(cls, /, obj: typing.Any) -> graph.GraphData: ... + def func( + self, + /, + function_name: str | None, + inputs: typing.Mapping[tierkreis_core.aliases.PortID, identifiers.ValueRef], + ) -> graph.CurriedNodeIndex: ... + def get_nodedef(self, /, node_idx: identifiers.NodeIndex) -> graph.NodeDef: ... + def graph_outputs( + self, / + ) -> dict[tierkreis_core.aliases.PortID, identifiers.NodeIndex] | None: ... + def if_else( + self, + /, + pred: identifiers.ValueRef, + if_true: identifiers.ValueRef, + if_false: identifiers.ValueRef, + ) -> graph.CurriedNodeIndex: ... + def input(self, /, name: str) -> identifiers.ValueRef: ... + def is_empty(self, /) -> bool: ... + def loop( + self, + /, + graph: identifiers.ExteriorRef | identifiers.ValueRef, + inputs: typing.Mapping[tierkreis_core.aliases.PortID, identifiers.ValueRef], + continue_port: tierkreis_core.aliases.PortID, + name: str | None = None, + ) -> graph.CurriedNodeIndex: ... + def map( + self, + /, + graph: identifiers.ExteriorRef | identifiers.ValueRef, + inputs: typing.Mapping[tierkreis_core.aliases.PortID, identifiers.ValueRef], + ) -> graph.CurriedNodeIndex: ... + def model_dump(self, /) -> typing.Any: ... + def model_dump_json(self, /) -> str: ... + @classmethod + def model_load(cls, /, obj: typing.Any) -> graph.GraphData: ... + @classmethod + def model_load_json(cls, /, s: str) -> graph.GraphData: ... + @property + def nodes(self, /) -> list[NodeDef]: ... + def output( + self, + /, + inputs: typing.Mapping[tierkreis_core.aliases.PortID, identifiers.ValueRef], + ) -> None: ... + def output_idx(self, /) -> identifiers.NodeIndex | None: ... + def outputs( + self, /, node_index: identifiers.NodeIndex + ) -> dict[tierkreis_core.aliases.PortID, identifiers.NodeIndex] | None: ... + def query_node_description(self, /, loc: location.Loc) -> graph.NodeDescription: ... + def remaining_inputs( + self, /, provided_inputs: typing.Set[tierkreis_core.aliases.PortID] + ) -> set[tierkreis_core.aliases.PortID]: ... + def to_dict(self, /) -> typing.Any: ... + +class NodeDef: + Func = tierkreis_core.nodes.NodeDef_Func + Eval = tierkreis_core.nodes.NodeDef_Eval + Loop = tierkreis_core.nodes.NodeDef_Loop + Map = tierkreis_core.nodes.NodeDef_Map + Const = tierkreis_core.nodes.NodeDef_Const + IfElse = tierkreis_core.nodes.NodeDef_IfElse + EagerIfElse = tierkreis_core.nodes.NodeDef_EagerIfElse + Input = tierkreis_core.nodes.NodeDef_Input + Output = tierkreis_core.nodes.NodeDef_Output + + def __eq__(self, /, other: object) -> bool: ... + def __ne__(self, /, other: object) -> bool: ... + def __repr__(self, /) -> str: ... + def __str__(self, /) -> str: ... + @property + def in_edges( + self, / + ) -> dict[ + tierkreis_core.aliases.PortID, identifiers.ValueRef | identifiers.ExteriorRef + ]: ... + @property + def inputs( + self, / + ) -> dict[ + tierkreis_core.aliases.PortID, identifiers.ValueRef | identifiers.ExteriorRef + ]: ... + def model_dump_json(self, /) -> str: ... + @classmethod + def model_load_json(cls, /, s: str) -> graph.NodeDef: ... + +class NodeDescription: + def __eq__(self, /, other: object) -> bool: ... + def __ne__(self, /, other: object) -> bool: ... + def __init__( + self, + /, + definition: graph.NodeDef, + outputs: typing.Mapping[ + tierkreis_core.aliases.PortID, identifiers.NodeIndex + ] = ..., + outer_graph: graph.GraphData | None = None, + ) -> None: ... + def __repr__(self, /) -> str: ... + def __str__(self, /) -> str: ... + @property + def definition(self, /) -> graph.NodeDef: ... + @classmethod + def from_dict(cls, /, obj: typing.Any) -> graph.NodeDescription: ... + def model_dump(self, /) -> typing.Any: ... + def model_dump_json(self, /) -> str: ... + @classmethod + def model_load(cls, /, obj: typing.Any) -> graph.NodeDescription: ... + @classmethod + def model_load_json(cls, /, s: str) -> graph.NodeDescription: ... + @property + def outer_graph(self, /) -> graph.GraphData | None: ... + @property + def outputs( + self, / + ) -> dict[tierkreis_core.aliases.PortID, identifiers.NodeIndex]: ... + def to_dict(self, /) -> typing.Any: ... + +def new_eval_root( + inputs: typing.Mapping[ + tierkreis_core.aliases.PortID, identifiers.ExteriorRef | identifiers.ValueRef + ], +) -> graph.NodeDef: ... +def __getattr__(name: str) -> _typeshed.Incomplete: ... diff --git a/tierkreis_core/python/tierkreis_core/_tierkreis_core/identifiers.pyi b/tierkreis_core/python/tierkreis_core/_tierkreis_core/identifiers.pyi new file mode 100644 index 000000000..44c3f26b9 --- /dev/null +++ b/tierkreis_core/python/tierkreis_core/_tierkreis_core/identifiers.pyi @@ -0,0 +1,36 @@ +import _typeshed +import tierkreis_core._tierkreis_core.identifiers as identifiers +import typing + +class ExteriorRef: + def __eq__(self, /, other: object) -> bool: ... + def __iter__(self, /) -> typing.Any: ... + def __ne__(self, /, other: object) -> bool: ... + def __init__(self, /, port_id: str) -> None: ... + def __repr__(self, /) -> str: ... + def __str__(self, /) -> str: ... + @property + def port_id(self, /) -> str: ... + +class NodeIndex: + def __eq__(self, /, other: object) -> bool: ... + def __hash__(self, /) -> int: ... + def __int__(self, /) -> int: ... + def __ne__(self, /, other: object) -> bool: ... + def __init__(self, /, val: int) -> None: ... + def __repr__(self, /) -> str: ... + def __str__(self, /) -> str: ... + +class ValueRef: + def __eq__(self, /, other: object) -> bool: ... + def __iter__(self, /) -> typing.Any: ... + def __ne__(self, /, other: object) -> bool: ... + def __init__(self, /, node_index: identifiers.NodeIndex, port_id: str) -> None: ... + def __repr__(self, /) -> str: ... + def __str__(self, /) -> str: ... + @property + def node_index(self, /) -> identifiers.NodeIndex: ... + @property + def port_id(self, /) -> str: ... + +def __getattr__(name: str) -> _typeshed.Incomplete: ... diff --git a/tierkreis_core/python/tierkreis_core/_tierkreis_core/location.pyi b/tierkreis_core/python/tierkreis_core/_tierkreis_core/location.pyi new file mode 100644 index 000000000..67558f118 --- /dev/null +++ b/tierkreis_core/python/tierkreis_core/_tierkreis_core/location.pyi @@ -0,0 +1,56 @@ +import _typeshed +import tierkreis_core.steps +import tierkreis_core._tierkreis_core.identifiers as identifiers +import tierkreis_core._tierkreis_core.location as location +import typing + +class Loc: + def L(self, /, loop_index: int) -> location.Loc: ... + def M(self, /, map_index: int) -> location.Loc: ... + def N(self, /, node_index: identifiers.NodeIndex | int) -> location.Loc: ... + def __eq__(self, /, other: object) -> bool: ... + def __fspath__(self, /) -> str: ... + @classmethod + def __get_pydantic_core_schema__( + cls, /, _source: typing.Any, _handler: typing.Any + ) -> typing.Any: ... + def __hash__(self, /) -> int: ... + def __ne__(self, /, other: object) -> bool: ... + def __init__(self, /, k: str = "-") -> None: ... + def __repr__(self, /) -> str: ... + def __str__(self, /) -> str: ... + @staticmethod + def _validate(value: location.Loc | str) -> location.Loc: ... + def extend_from_ref( + self, /, value_ref: identifiers.ExteriorRef | identifiers.ValueRef + ) -> location.Loc: ... + def exterior(self, /) -> location.Loc: ... + @staticmethod + def from_steps(steps: typing.Sequence[NodeStep]) -> location.Loc: ... + def is_root_loc(self, /) -> bool: ... + def last_step_exterior(self, /) -> bool: ... + def parent(self, /) -> Loc | None: ... + def partial_locs(self, /) -> typing.Any: ... + def peek_index(self, /) -> int | None: ... + def pop_first(self, /) -> tuple[NodeStep, Loc]: ... + def pop_last(self, /) -> tuple[NodeStep, Loc]: ... + def startswith(self, /, other: location.Loc) -> bool: ... + def steps(self, /) -> list[Loc]: ... + +class NodeStep: + Root = tierkreis_core.steps.NodeStep_Root + Node = tierkreis_core.steps.NodeStep_Node + Loop = tierkreis_core.steps.NodeStep_Loop + Map = tierkreis_core.steps.NodeStep_Map + Exterior = tierkreis_core.steps.NodeStep_Exterior + + def __eq__(self, /, other: object) -> bool: ... + def __hash__(self, /) -> int: ... + def __iter__(self, /) -> typing.Any: ... + def __ne__(self, /, other: object) -> bool: ... + def __init__(self, /, steps: str) -> None: ... + def __repr__(self, /) -> str: ... + def __str__(self, /) -> str: ... + def is_root(self, /) -> bool: ... + +def __getattr__(name: str) -> _typeshed.Incomplete: ... diff --git a/tierkreis_core/python/tierkreis_core/_tierkreis_core/value.pyi b/tierkreis_core/python/tierkreis_core/_tierkreis_core/value.pyi new file mode 100644 index 000000000..b88c3a5f3 --- /dev/null +++ b/tierkreis_core/python/tierkreis_core/_tierkreis_core/value.pyi @@ -0,0 +1,3 @@ +import _typeshed + +def __getattr__(name: str) -> _typeshed.Incomplete: ... diff --git a/tierkreis_core/python/tierkreis_core/aliases.py b/tierkreis_core/python/tierkreis_core/aliases.py new file mode 100644 index 000000000..31c152472 --- /dev/null +++ b/tierkreis_core/python/tierkreis_core/aliases.py @@ -0,0 +1,15 @@ +"""Handwritten stub for alias types that cannot be generated by pyO3 introspection. + +See: https://github.com/PyO3/pyo3/issues/5137 for progress. + +""" + +from typing import Mapping, Sequence + +from tierkreis_core import GraphData + +type PortID = str +# Constant values +type Value = ( + int | float | bool | str | bytes | Sequence[Value] | Mapping[str, Value] | GraphData +) diff --git a/tierkreis_core/python/tierkreis_core/aliases.pyi b/tierkreis_core/python/tierkreis_core/aliases.pyi new file mode 100644 index 000000000..1d07a748b --- /dev/null +++ b/tierkreis_core/python/tierkreis_core/aliases.pyi @@ -0,0 +1,14 @@ +"""Type aliases for documentation and typechecking purposes.""" + +from typing import Mapping, Sequence + +from tierkreis_core._tierkreis_core.graph import GraphData + +# Identifiers for "ports" out or into nodes in a Graph. +type PortID = str +# Values that are acceptable for use in a Const node of a Graph. +# +# Strictly a subset of types like `tierkreis.PType`. +type Value = ( + int | float | bool | str | bytes | Sequence[Value] | Mapping[str, Value] | GraphData +) diff --git a/tierkreis_core/python/tierkreis_core/nodes.pyi b/tierkreis_core/python/tierkreis_core/nodes.pyi new file mode 100644 index 000000000..f19202d27 --- /dev/null +++ b/tierkreis_core/python/tierkreis_core/nodes.pyi @@ -0,0 +1,135 @@ +"""Handwritten stub for Node enum types that cannot be generated by pyO3 introspection. + +See: https://github.com/PyO3/pyo3/issues/5137 for progress. + +""" + +from typing import Mapping +import tierkreis_core +import tierkreis_core._tierkreis_core.identifiers as identifiers +from tierkreis_core import NodeDef + +class NodeDef_Func(NodeDef): + def __init__( + self, + name: str, + inputs: Mapping[ + tierkreis_core.aliases.PortID, + identifiers.ExteriorRef | identifiers.ValueRef, + ], + ) -> None: ... + @property + def name(self, /) -> str: ... + @property + def inputs( + self, / + ) -> dict[ + tierkreis_core.aliases.PortID, identifiers.ExteriorRef | identifiers.ValueRef + ]: ... + +class NodeDef_Eval(NodeDef): + def __init__( + self, + body: identifiers.ExteriorRef | identifiers.ValueRef, + inputs: Mapping[ + tierkreis_core.aliases.PortID, + identifiers.ExteriorRef | identifiers.ValueRef, + ], + ) -> None: ... + @property + def body(self, /) -> identifiers.ExteriorRef | identifiers.ValueRef: ... + @property + def inputs( + self, / + ) -> dict[ + tierkreis_core.aliases.PortID, identifiers.ExteriorRef | identifiers.ValueRef + ]: ... + +class NodeDef_Loop(NodeDef): + def __init__( + self, + body: identifiers.ExteriorRef | identifiers.ValueRef, + inputs: Mapping[ + tierkreis_core.aliases.PortID, + identifiers.ExteriorRef | identifiers.ValueRef, + ], + continue_port: tierkreis_core.aliases.PortID, + name: str | None = None, + ) -> None: ... + @property + def name(self, /) -> str | None: ... + @property + def body(self, /) -> identifiers.ExteriorRef | identifiers.ValueRef: ... + @property + def inputs( + self, / + ) -> dict[ + tierkreis_core.aliases.PortID, identifiers.ExteriorRef | identifiers.ValueRef + ]: ... + @property + def continue_port(self, /) -> tierkreis_core.aliases.PortID: ... + +class NodeDef_Map(NodeDef): + def __init__( + self, + body: identifiers.ExteriorRef | identifiers.ValueRef, + inputs: Mapping[ + tierkreis_core.aliases.PortID, + identifiers.ExteriorRef | identifiers.ValueRef, + ], + ) -> None: ... + @property + def body(self, /) -> identifiers.ExteriorRef | identifiers.ValueRef: ... + @property + def inputs( + self, / + ) -> dict[ + tierkreis_core.aliases.PortID, identifiers.ExteriorRef | identifiers.ValueRef + ]: ... + +class NodeDef_Const(NodeDef): + def __init__(self, value: tierkreis_core.aliases.Value) -> None: ... + @property + def value(self, /) -> tierkreis_core.aliases.Value: ... + +class NodeDef_IfElse(NodeDef): + def __init__( + self, + pred: identifiers.ValueRef, + if_true: identifiers.ValueRef, + if_false: identifiers.ValueRef, + ) -> None: ... + @property + def pred(self, /) -> identifiers.ValueRef: ... + @property + def if_true(self, /) -> identifiers.ValueRef: ... + @property + def if_false(self, /) -> identifiers.ValueRef: ... + +class NodeDef_EagerIfElse(NodeDef): + def __init__( + self, + pred: identifiers.ValueRef, + if_true: identifiers.ValueRef, + if_false: identifiers.ValueRef, + ) -> None: ... + @property + def pred(self, /) -> identifiers.ValueRef: ... + @property + def if_true(self, /) -> identifiers.ValueRef: ... + @property + def if_false(self, /) -> identifiers.ValueRef: ... + +class NodeDef_Input(NodeDef): + def __init__(self, name: tierkreis_core.aliases.PortID) -> None: ... + @property + def name(self, /) -> tierkreis_core.aliases.PortID: ... + +class NodeDef_Output(NodeDef): + def __init__( + self, outputs: Mapping[tierkreis_core.aliases.PortID, identifiers.ValueRef] + ) -> None: ... + @property + def outputs( + self, / + ) -> dict[tierkreis_core.aliases.PortID, identifiers.ValueRef]: ... diff --git a/tierkreis_core/python/tierkreis_core/py.typed b/tierkreis_core/python/tierkreis_core/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/tierkreis_core/python/tierkreis_core/steps.pyi b/tierkreis_core/python/tierkreis_core/steps.pyi new file mode 100644 index 000000000..e395ab945 --- /dev/null +++ b/tierkreis_core/python/tierkreis_core/steps.pyi @@ -0,0 +1,23 @@ +"""Handwritten stub for Node enum types that cannot be generated by pyO3 introspection. + +See: https://github.com/PyO3/pyo3/issues/5137 for progress. + +""" + +from tierkreis_core import NodeStep, NodeIndex + +class NodeStep_Root(NodeStep): ... + +class NodeStep_Node(NodeStep): + @property + def node_index(self, /) -> NodeIndex: ... + +class NodeStep_Loop(NodeStep): + @property + def loop_index(self, /) -> int: ... + +class NodeStep_Map(NodeStep): + @property + def map_index(self, /) -> int: ... + +class NodeStep_Exterior(NodeStep): ... diff --git a/tierkreis_core/rust/bin/tierkreis-core-stubs-gen.rs b/tierkreis_core/rust/bin/tierkreis-core-stubs-gen.rs new file mode 100644 index 000000000..facc907f0 --- /dev/null +++ b/tierkreis_core/rust/bin/tierkreis-core-stubs-gen.rs @@ -0,0 +1,126 @@ +use std::path::PathBuf; + +use regex::Regex; + +// `crate::graph::NodeDef` has extra class attributes that cannot be generated by +// `pyo3_introspection`. So we patch them in manually from a handwritten stub. +const NODEDEF_CLASS_PATCH: &'static str = " +class NodeDef: + Func = tierkreis_core.nodes.NodeDef_Func + Eval = tierkreis_core.nodes.NodeDef_Eval + Loop = tierkreis_core.nodes.NodeDef_Loop + Map = tierkreis_core.nodes.NodeDef_Map + Const = tierkreis_core.nodes.NodeDef_Const + IfElse = tierkreis_core.nodes.NodeDef_IfElse + EagerIfElse = tierkreis_core.nodes.NodeDef_EagerIfElse + Input = tierkreis_core.nodes.NodeDef_Input + Output = tierkreis_core.nodes.NodeDef_Output +"; + +// `crate::graph::NodeStep` has extra class attributes that cannot be generated by +// `pyo3_introspection`. So we patch them in manually from a handwritten stub. +const NODESTEP_CLASS_PATCH: &'static str = " +class NodeStep: + Root = tierkreis_core.steps.NodeStep_Root + Node = tierkreis_core.steps.NodeStep_Node + Loop = tierkreis_core.steps.NodeStep_Loop + Map = tierkreis_core.steps.NodeStep_Map + Exterior = tierkreis_core.steps.NodeStep_Exterior +"; + +pub fn main() { + // This assumes the package is installed as editable. + // + // TODO: perhaps check for venv variable instead? + let module = pyo3_introspection::introspect_cdylib( + "./python/tierkreis_core/_tierkreis_core.abi3.so", + "_tierkreis_core", + ) + .unwrap(); + let stubs = pyo3_introspection::module_stub_files(&module); + + for (stub_name, stub) in &stubs { + // The stub file adds types new() methods and not constructors + // + // This is technically correct but doesn't play nicely with pyright, so patch them for now. + let mut patched_stub = stub.replace("__new__(cls", "__init__(self"); + + for (stub_name, _) in &stubs { + let module_name = stub_name.file_stem().unwrap().to_str().unwrap(); + patched_stub = patched_stub.replace( + &format!("import {}", module_name), + &format!( + "import tierkreis_core._tierkreis_core.{} as {}", + module_name, module_name + ), + ); + } + + if stub_name.file_stem().unwrap().to_str().unwrap() == "graph" { + // Add an extra import for the handwritten stub. + patched_stub = patched_stub.replace( + "import tierkreis_core.aliases", + "import tierkreis_core.aliases\nimport tierkreis_core.nodes", + ); + + patched_stub = patched_stub.replace("class NodeDef:", NODEDEF_CLASS_PATCH); + + // Extra patches for erroneous typing.Any on properties. + + // // Patch nodes on GraphData + patched_stub = patched_stub.replace( + "def nodes(self, /) -> typing.Any", + "def nodes(self, /) -> list[NodeDef]", + ); + + // // Patch inputs on NodeDef + patched_stub = patched_stub.replace( + "def inputs(self, /) -> typing.Any", + "def inputs(self, /) -> dict[tierkreis_core.aliases.PortID, identifiers.ValueRef | identifiers.ExteriorRef]", + ); + + // // Patch in_edges on NodeDef + patched_stub = patched_stub.replace( + "def in_edges(self, /) -> typing.Any", + "def in_edges(self, /) -> dict[tierkreis_core.aliases.PortID, identifiers.ValueRef | identifiers.ExteriorRef]", + ); + + // // Patch outer_graph on NodeDescription + patched_stub = patched_stub.replace( + "def outer_graph(self, /) -> typing.Any", + "def outer_graph(self, /) -> graph.GraphData | None", + ); + + // Patch outputs on NodeDescription + patched_stub = patched_stub.replace( + "def outputs(self, /) -> typing.Any", + "def outputs(self, /) -> dict[tierkreis_core.aliases.PortID, identifiers.NodeIndex]", + ); + } + + if stub_name.file_stem().unwrap().to_str().unwrap() == "location" { + // Add an extra import for the handwritten stub. + patched_stub = patched_stub.replace( + "import _typeshed", + "import _typeshed\nimport tierkreis_core.steps", + ); + + patched_stub = patched_stub.replace("class NodeStep:", NODESTEP_CLASS_PATCH); + } + + // Python expects the type of the target in __eq__ to actually be object + let eq_regex = Regex::new(r"__eq__\(self, /, other: \w+\.\w+").unwrap(); + let patched_stub = eq_regex.replace_all(&patched_stub, "__eq__(self, /, other: object"); + + // Python expects the type of the target in __ne__ to actually be object + let eq_regex = Regex::new(r"__ne__\(self, /, other: \w+\.\w+").unwrap(); + let patched_stub = eq_regex.replace_all(&patched_stub, "__ne__(self, /, other: object"); + + let mut path = PathBuf::new(); + path.push("python"); + path.push("tierkreis_core"); + path.push("_tierkreis_core"); + path.push(stub_name); + std::fs::write(path, &*patched_stub).unwrap(); + } +} diff --git a/tierkreis_core/rust/graph.rs b/tierkreis_core/rust/graph.rs new file mode 100644 index 000000000..f836b7b5d --- /dev/null +++ b/tierkreis_core/rust/graph.rs @@ -0,0 +1,746 @@ +use std::collections::{HashMap, HashSet}; + +use indexmap::{IndexMap, IndexSet}; +use pyo3::{ + exceptions::PyValueError, + prelude::*, + types::{PyDict, PyType}, +}; +use pythonize::{depythonize, pythonize}; +use serde::{Deserialize, Serialize}; + +/// Graph utilities +#[pymodule(submodule)] +pub mod graph { + use pyo3::exceptions::PyIndexError; + + use crate::{ + identifiers::identifiers::{ExteriorOrValueRef, ExteriorRef, NodeIndex, PortID, ValueRef}, + location::location::{Loc, NodeStep}, + value::value::Value, + }; + + use super::*; + + /// Hack: workaround for https://github.com/PyO3/pyo3/issues/759 + #[pymodule_init] + fn init(m: &Bound<'_, PyModule>) -> PyResult<()> { + Python::attach(|py| { + py.import("sys")? + .getattr("modules")? + .set_item("tierkreis_core._tierkreis_core.graph", m) + }) + } + + /// new_eval_root is a utility function to create a new eval node that + /// references the outer scope for its body. + /// + // Assumes the values have been written to storage using the same names + #[pyfunction(signature = (inputs: "typing.Mapping[tierkreis_core.aliases.PortID, identifiers.ExteriorRef | identifiers.ValueRef]"))] + pub fn new_eval_root(inputs: HashMap) -> NodeDef { + let inputs_refs = inputs + .into_iter() + .filter(|(port_id, _)| port_id != "body") + .collect(); + NodeDef::Eval { + body: ExteriorOrValueRef::Exterior(ExteriorRef("body".to_string())), + inputs: inputs_refs, + } + } + + #[pyclass(eq, frozen)] + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] + pub enum NodeDef { + Func { + name: String, + inputs: IndexMap, + }, + Eval { + body: ExteriorOrValueRef, + inputs: IndexMap, + }, + Loop { + name: Option, + body: ExteriorOrValueRef, + inputs: IndexMap, + continue_port: PortID, + }, + Map { + body: ExteriorOrValueRef, + inputs: IndexMap, + }, + Const { + value: Value, + }, + IfElse { + pred: ValueRef, + if_true: ValueRef, + if_false: ValueRef, + }, + EagerIfElse { + pred: ValueRef, + if_true: ValueRef, + if_false: ValueRef, + }, + Input { + name: PortID, + }, + Output { + inputs: IndexMap, + }, + } + + impl std::fmt::Display for NodeDef { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Func { name, inputs } => write!(f, "Func('{}', {:?})", name, inputs), + Self::Eval { inputs, .. } => write!(f, "Eval(..., {:?})", inputs), + Self::Loop { + inputs, + continue_port, + .. + } => write!(f, "Loop(..., {:?}, {})", inputs, continue_port), + Self::Map { inputs, .. } => write!(f, "Map(..., {:?})", inputs), + Self::Const { value } => write!(f, "Const({:?})", value), + Self::IfElse { + pred, + if_true, + if_false, + } => write!(f, "IfElse({}, {}, {})", pred, if_true, if_false), + Self::EagerIfElse { + pred, + if_true, + if_false, + } => write!(f, "EagerIfElse({}, {}, {})", pred, if_true, if_false), + Self::Input { name } => write!(f, "Input('{}')", name), + Self::Output { inputs } => write!(f, "Output({:?})", inputs), + } + } + } + + #[pymethods] + impl NodeDef { + /// Get the `inputs` attribute for a Node. This is different to + /// `in_edges` as it only considers the edges under the `inputs` + /// attribute and not things like the body of Eval nodes which + /// is considered a different kind of edge. + #[getter] + pub fn inputs(&self) -> IndexMap { + match self { + Self::Eval { inputs, .. } => inputs.clone(), + Self::Loop { inputs, .. } => inputs.clone(), + Self::Map { inputs, .. } => inputs.clone(), + Self::Func { inputs, .. } => inputs.clone(), + Self::Const { .. } => IndexMap::new(), + Self::IfElse { .. } => IndexMap::new(), + Self::EagerIfElse { .. } => IndexMap::new(), + Self::Input { .. } => IndexMap::new(), + Self::Output { inputs, .. } => inputs + .iter() + .map(|x| (x.0.clone(), ExteriorOrValueRef::Value(x.1.clone()))) + .collect(), + } + } + + /// Get the `in_edges` attribute for a Node. This is different to + /// `inputs` as it also considers the edges connected to the body + /// port of Eval nodes which does not come under the `inputs` attribute. + #[getter] + pub fn in_edges(&self) -> IndexMap { + match self { + Self::Eval { inputs, body } => { + let mut inputs = inputs.clone(); + inputs.insert("body".to_string(), body.clone()); + inputs + } + Self::Loop { inputs, body, .. } => { + let mut inputs = inputs.clone(); + inputs.insert("body".to_string(), body.clone()); + inputs + } + Self::Map { inputs, body, .. } => { + let mut inputs = inputs.clone(); + inputs.insert("body".to_string(), body.clone()); + inputs + } + Self::Const { .. } => IndexMap::new(), + Self::IfElse { pred, .. } => { + let mut inputs = IndexMap::new(); + inputs.insert("pred".to_string(), ExteriorOrValueRef::Value(pred.clone())); + inputs + } + Self::EagerIfElse { + pred, + if_true, + if_false, + } => { + let mut inputs = IndexMap::new(); + inputs.insert("pred".to_string(), ExteriorOrValueRef::Value(pred.clone())); + inputs.insert( + "body_true".to_string(), + ExteriorOrValueRef::Value(if_true.clone()), + ); + inputs.insert( + "body_false".to_string(), + ExteriorOrValueRef::Value(if_false.clone()), + ); + inputs + } + Self::Func { inputs, .. } => inputs.clone(), + Self::Input { .. } => IndexMap::new(), + Self::Output { inputs, .. } => inputs + .iter() + .map(|x| (x.0.clone(), ExteriorOrValueRef::Value(x.1.clone()))) + .collect(), + } + } + + pub fn model_dump_json(&self) -> String { + serde_json::to_string(self).unwrap() + } + + #[classmethod] + pub fn model_load_json(_cls: &Bound<'_, PyType>, s: &str) -> Self { + serde_json::from_str(s).unwrap() + } + + pub fn __repr__(&self) -> String { + format!("{}", self) + } + + pub fn __str__(&self) -> String { + format!("{}", self) + } + } + + #[pyclass] + pub struct CurriedNodeIndex { + node_index: NodeIndex, + } + + #[pymethods] + impl CurriedNodeIndex { + fn __call__(&self, port: PortID) -> ValueRef { + ValueRef(self.node_index.0, port) + } + } + + #[pyclass(eq)] + #[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq)] + pub struct GraphData { + // First argument is the node definition, followed + // by a set of which of the output ports have been + // connected to other ports. + nodes: Vec<(NodeDef, IndexMap)>, + fixed_inputs: IndexMap, + graph_inputs: IndexSet, + output_idx: Option, + } + + #[pymethods] + impl GraphData { + #[new] + pub fn new() -> Result { + Ok(Self { + ..Default::default() + }) + } + + pub fn is_empty(&self) -> bool { + self.nodes.is_empty() + } + + pub fn get_nodedef(&self, node_idx: NodeIndex) -> PyResult { + self.nodes + .get(node_idx.0) + .map(|x| x.0.clone()) + .ok_or(PyIndexError::new_err( + "Node not found in graph with that index", + )) + } + + /// Retrieve the nodes in the graph as a list. + #[getter] + pub fn nodes(&self) -> Vec { + self.nodes.iter().map(|x| &x.0).cloned().collect() + } + + #[getter] + pub fn fixed_inputs(&self) -> IndexMap { + self.fixed_inputs.clone() + } + + /// Get the `NodeIndex` of the output node if it exists. + /// + /// For historical reasons this is a function rather + /// than a getter. + #[pyo3(signature = () -> "identifiers.NodeIndex | None")] + pub fn output_idx(&self) -> Option { + self.output_idx.clone() + } + + pub fn input(&mut self, name: &str) -> ValueRef { + let idx = self.add_node(NodeDef::Input { + name: name.to_string(), + }); + self.graph_inputs.insert(name.to_string()); + + ValueRef(idx.0, name.to_string()) + } + + /// Add a Const node to the graph. + #[pyo3(signature = (value: "tierkreis_core.aliases.Value"))] + pub fn r#const(&mut self, value: Value) -> ValueRef { + let idx = self.add_node(NodeDef::Const { value }); + + ValueRef(idx.0, "value".to_string()) + } + + #[pyo3(signature = ( + function_name: "str | None", + inputs: "typing.Mapping[tierkreis_core.aliases.PortID, identifiers.ValueRef]", + ))] + pub fn func( + &mut self, + function_name: &str, + inputs: IndexMap, + ) -> CurriedNodeIndex { + let node_index = self.add_node(NodeDef::Func { + name: function_name.to_string(), + inputs: inputs + .iter() + .map(|(port_id, edge_value_ref)| { + ( + port_id.clone(), + ExteriorOrValueRef::Value(edge_value_ref.clone().clone()), + ) + }) + .collect(), + }); + self.connect_inputs(node_index, &inputs); + + CurriedNodeIndex { node_index } + } + + #[pyo3(signature = ( + graph: "identifiers.ExteriorRef | identifiers.ValueRef", + inputs: "typing.Mapping[tierkreis_core.aliases.PortID, identifiers.ValueRef]", + ))] + pub fn eval( + &mut self, + graph: ExteriorOrValueRef, + inputs: IndexMap, + ) -> CurriedNodeIndex { + let node_index = self.add_node(NodeDef::Eval { + body: graph, + inputs: inputs + .iter() + .map(|(port_id, edge_value_ref)| { + ( + port_id.clone(), + ExteriorOrValueRef::Value(edge_value_ref.clone().clone()), + ) + }) + .collect(), + }); + self.connect_inputs(node_index, &inputs); + + CurriedNodeIndex { node_index } + } + + #[pyo3(signature = ( + graph: "identifiers.ExteriorRef | identifiers.ValueRef", + inputs: "typing.Mapping[tierkreis_core.aliases.PortID, identifiers.ValueRef]", + continue_port: "tierkreis_core.aliases.PortID", + name: "str | None" = None + ))] + pub fn r#loop( + &mut self, + graph: ExteriorOrValueRef, + inputs: IndexMap, + continue_port: PortID, + name: Option, + ) -> CurriedNodeIndex { + let node_index = self.add_node(NodeDef::Loop { + name, + body: graph, + inputs: inputs + .iter() + .map(|(port_id, edge_value_ref)| { + ( + port_id.clone(), + ExteriorOrValueRef::Value(edge_value_ref.clone().clone()), + ) + }) + .collect(), + continue_port, + }); + self.connect_inputs(node_index, &inputs); + + CurriedNodeIndex { node_index } + } + + #[pyo3(signature = ( + graph: "identifiers.ExteriorRef | identifiers.ValueRef", + inputs: "typing.Mapping[tierkreis_core.aliases.PortID, identifiers.ValueRef]", + ))] + pub fn map( + &mut self, + graph: ExteriorOrValueRef, + inputs: IndexMap, + ) -> CurriedNodeIndex { + let node_index = self.add_node(NodeDef::Map { + body: graph, + inputs: inputs + .iter() + .map(|(port_id, edge_value_ref)| { + ( + port_id.clone(), + ExteriorOrValueRef::Value(edge_value_ref.clone().clone()), + ) + }) + .collect(), + }); + self.connect_inputs(node_index, &inputs); + + CurriedNodeIndex { node_index } + } + + pub fn if_else( + &mut self, + pred: ValueRef, + if_true: ValueRef, + if_false: ValueRef, + ) -> CurriedNodeIndex { + let node_index = self.add_node(NodeDef::IfElse { + pred: pred.clone(), + if_true: if_true.clone(), + if_false: if_false.clone(), + }); + + self.nodes[pred.0].1.insert(pred.1, node_index); + self.nodes[if_true.0].1.insert(if_true.1, node_index); + self.nodes[if_false.0].1.insert(if_false.1, node_index); + + CurriedNodeIndex { node_index } + } + + pub fn eager_if_else( + &mut self, + pred: ValueRef, + if_true: ValueRef, + if_false: ValueRef, + ) -> CurriedNodeIndex { + let node_index = self.add_node(NodeDef::EagerIfElse { + pred: pred.clone(), + if_true: if_true.clone(), + if_false: if_false.clone(), + }); + + self.nodes[pred.0].1.insert(pred.1, node_index); + self.nodes[if_true.0].1.insert(if_true.1, node_index); + self.nodes[if_false.0].1.insert(if_false.1, node_index); + + CurriedNodeIndex { node_index } + } + + #[pyo3(signature = ( + inputs: "typing.Mapping[tierkreis_core.aliases.PortID, identifiers.ValueRef]", + ) -> "None")] + pub fn output(&mut self, inputs: IndexMap) -> PyResult<()> { + if self.output_idx.is_some() { + return Err(PyValueError::new_err("Output already set")); + } + + let node_index = self.add_node(NodeDef::Output { + inputs: inputs.iter().map(|(k, v)| (k.clone(), v.clone())).collect(), + }); + self.connect_inputs(node_index, &inputs); + self.output_idx = Some(node_index); + + Ok(()) + } + + #[pyo3(signature = (provided_inputs: "typing.Set[tierkreis_core.aliases.PortID]") -> "set[tierkreis_core.aliases.PortID]")] + pub fn remaining_inputs(&self, provided_inputs: HashSet) -> HashSet { + let fixed_inputs: HashSet = self.fixed_inputs.keys().cloned().collect(); + if fixed_inputs.intersection(&provided_inputs).next().is_some() { + unimplemented!(); + } + + let actual_inputs: IndexSet = + fixed_inputs.union(&provided_inputs).cloned().collect(); + self.graph_inputs + .difference(&actual_inputs) + .cloned() + .collect() + } + + #[pyo3(signature = (node_index: "identifiers.NodeIndex") -> "dict[tierkreis_core.aliases.PortID, identifiers.NodeIndex] | None")] + pub fn outputs(&self, node_index: NodeIndex) -> Option> { + self.nodes.get(node_index.0).map(|(_, outputs)| { + outputs + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect() + }) + } + + #[pyo3(signature = () -> "dict[tierkreis_core.aliases.PortID, identifiers.NodeIndex] | None")] + pub fn graph_outputs(&self) -> Option> { + self.output_idx + .and_then(|idx| self.nodes.get(idx.0)) + .and_then(|(node, _)| match node { + NodeDef::Output { inputs } => Some( + inputs + .iter() + .map(|(k, v)| (k.clone(), v.node_index())) + .collect(), + ), + _ => None, + }) + } + + /// Query a NodeDescription from a Loc (which describes a location on the graph.) + /// + /// Useful for visualisation and debugging. + pub fn query_node_description(&self, loc: Loc) -> PyResult { + if self.is_empty() { + return Err(PyValueError::new_err("Cannot query and empty graph")); + } + if loc.is_root_loc() { + return Ok(NodeDescription { + definition: new_eval_root(HashMap::new()), + outputs: HashMap::new(), + outer_graph: Some(self.clone()), + }); + } + + let (step, remaining) = loc.pop_first()?; + let (nodedef, outputs) = match step { + NodeStep::Node { node_index } => self + .nodes + .get(node_index.0) + .ok_or(PyIndexError::new_err("Node not found with that index"))?, + NodeStep::Exterior {} => { + return Ok(NodeDescription { + definition: new_eval_root(HashMap::new()), + outputs: HashMap::new(), + outer_graph: Some(self.clone()), + }) + } + _ => { + return Err(PyValueError::new_err(format!( + "Malformed Loc: First node is not a Node step: {}", + step + ))) + } + }; + + match nodedef { + NodeDef::Eval { + body: ExteriorOrValueRef::Value(value_ref), + .. + } => { + let connected_nodedef = self.get_nodedef(value_ref.node_index())?; + let const_graph = match connected_nodedef { + NodeDef::Const { + value: Value::Graph(graph), + } => graph, + _ => { + return Err(PyValueError::new_err( + "Const node connected to body port does not contain a graph", + )) + } + }; + const_graph + .query_node_description(remaining) + .map(|mut desc| { + desc.outer_graph = Some(const_graph); + desc + }) + } + NodeDef::Loop { + body: ExteriorOrValueRef::Value(value_ref), + inputs, + .. + } + | NodeDef::Map { + body: ExteriorOrValueRef::Value(value_ref), + inputs, + .. + } => { + let connected_nodedef = self.get_nodedef(value_ref.node_index())?; + let const_graph = match connected_nodedef { + NodeDef::Const { + value: Value::Graph(graph), + } => graph, + _ => { + return Err(PyValueError::new_err( + "Const node connected to body port does not contain a graph", + )) + } + }; + let (_, remaining) = remaining.pop_first()?; + if remaining.steps().len() < 2 { + return Ok(NodeDescription { + definition: NodeDef::Eval { + body: ExteriorOrValueRef::Exterior(ExteriorRef("body".to_string())), + inputs: inputs.clone(), + }, + outputs: outputs + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(), + outer_graph: Some(const_graph.clone()), + }); + } + const_graph + .query_node_description(remaining) + .map(|mut desc| { + desc.outer_graph = Some(const_graph); + desc + }) + } + _ => Ok(NodeDescription { + definition: nodedef.clone(), + outputs: outputs + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(), + outer_graph: Some(self.clone()), + }), + } + } + + pub fn to_dict<'py>(&self, py: Python<'py>) -> PyResult> { + Ok(pythonize(py, self)?.extract()?) + } + + #[classmethod] + pub fn from_dict(_cls: &Bound<'_, PyType>, obj: Bound) -> PyResult { + Ok(depythonize(&obj)?) + } + + pub fn model_dump<'py>(&self, py: Python<'py>) -> PyResult> { + self.to_dict(py) + } + + #[classmethod] + pub fn model_load(cls: &Bound<'_, PyType>, obj: Bound) -> PyResult { + Self::from_dict(cls, obj) + } + + pub fn model_dump_json(&self) -> String { + serde_json::to_string(self).unwrap() + } + + #[classmethod] + pub fn model_load_json(_cls: &Bound<'_, PyType>, s: &str) -> Self { + serde_json::from_str(s).unwrap() + } + } + + impl GraphData { + fn connect_inputs( + &mut self, + new_node_index: NodeIndex, + inputs: &IndexMap, + ) { + for (_, ValueRef(idx, port)) in inputs { + self.nodes[*idx].1.insert(port.clone(), new_node_index); + } + } + + fn add_node(&mut self, node: NodeDef) -> NodeIndex { + // Nodes are 0-indexed so get the length before adding the new node. + let node_index = self.nodes.len(); + self.nodes.push((node, IndexMap::new())); + NodeIndex(node_index) + } + } + + /// An enriched version of NodeDef that contains the outputs + /// and potentially the body of a sub-graph if available. + /// + /// Useful for visualisation and debugging. + #[pyclass(eq)] + #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] + pub struct NodeDescription { + #[pyo3(get)] + pub definition: NodeDef, + #[pyo3(get)] + pub outputs: HashMap, + // If the node is an eval, map or loop it may include an + // embedded graph if the parent graph contains a const + // node that is connected this node's body port. + #[pyo3(get)] + pub outer_graph: Option, + } + + #[pymethods] + impl NodeDescription { + #[new] + #[pyo3(signature = ( + definition: "graph.NodeDef", + outputs: "typing.Mapping[tierkreis_core.aliases.PortID, identifiers.NodeIndex]" = IndexMap::new(), + outer_graph: "graph.GraphData | None" = None, + ))] + pub fn new( + definition: NodeDef, + outputs: IndexMap, + outer_graph: Option, + ) -> PyResult { + Ok(Self { + definition, + outputs: outputs.into_iter().collect(), + outer_graph, + }) + } + + pub fn __repr__(&self) -> String { + format!("{}", self) + } + + pub fn __str__(&self) -> String { + format!("{}", self) + } + + pub fn to_dict<'py>(&self, py: Python<'py>) -> PyResult> { + Ok(pythonize(py, self)?.extract()?) + } + + #[classmethod] + pub fn from_dict(_cls: &Bound<'_, PyType>, obj: Bound) -> PyResult { + Ok(depythonize(&obj)?) + } + + pub fn model_dump<'py>(&self, py: Python<'py>) -> PyResult> { + self.to_dict(py) + } + + #[classmethod] + pub fn model_load(cls: &Bound<'_, PyType>, obj: Bound) -> PyResult { + Self::from_dict(cls, obj) + } + + pub fn model_dump_json(&self) -> String { + serde_json::to_string(self).unwrap() + } + + #[classmethod] + pub fn model_load_json(_cls: &Bound<'_, PyType>, s: &str) -> Self { + serde_json::from_str(s).unwrap() + } + } + + impl std::fmt::Display for NodeDescription { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "NodeDescription({}, {:?}, {:?})", + self.definition, self.outputs, self.outer_graph + ) + } + } +} diff --git a/tierkreis_core/rust/identifiers.rs b/tierkreis_core/rust/identifiers.rs new file mode 100644 index 000000000..554419470 --- /dev/null +++ b/tierkreis_core/rust/identifiers.rs @@ -0,0 +1,151 @@ +use pyo3::{prelude::*, types::PyIterator}; +use serde::{Deserialize, Serialize}; + +#[pymodule(submodule)] +pub mod identifiers { + use super::*; + + /// Hack: workaround for https://github.com/PyO3/pyo3/issues/759 + #[pymodule_init] + fn init(m: &Bound<'_, PyModule>) -> PyResult<()> { + Python::attach(|py| { + py.import("sys")? + .getattr("modules")? + .set_item("tierkreis_core._tierkreis_core.identifiers", m) + }) + } + + pub type PortID = String; + + #[pyclass(eq, frozen, hash)] + #[derive(Clone, Copy, PartialEq, Serialize, Deserialize, Hash, Debug)] + pub struct NodeIndex(pub usize); + + impl std::fmt::Display for NodeIndex { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "NodeIndex({})", self.0)?; + Ok(()) + } + } + + #[pymethods] + impl NodeIndex { + #[new] + pub fn new(val: usize) -> Self { + Self(val) + } + + pub fn __repr__(&self) -> String { + format!("{}", self) + } + + pub fn __str__(&self) -> String { + self.0.to_string() + } + + pub fn __int__(&self) -> PyResult { + let x = self.0.try_into()?; + Ok(x) + } + } + + /// A reference to a value on an edge of a graph. + #[pyclass(eq, frozen, generic)] + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] + pub struct ValueRef(pub usize, pub String); + + impl std::fmt::Display for ValueRef { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "ValueRef({}, '{}')", self.0, self.1)?; + Ok(()) + } + } + + #[pymethods] + impl ValueRef { + #[new] + pub fn new(node_index: NodeIndex, port_id: PortID) -> Self { + Self(node_index.0, port_id) + } + + #[getter] + pub fn node_index(&self) -> NodeIndex { + NodeIndex(self.0) + } + + #[getter] + pub fn port_id(&self) -> PortID { + self.1.clone() + } + + pub fn __iter__<'py>(&self, py: Python<'py>) -> PyResult> { + PyIterator::from_object( + &(NodeIndex(self.0), self.1.clone()) + .into_pyobject(py)? + .into_any(), + ) + } + + pub fn __repr__(&self) -> String { + format!("{}", self) + } + + pub fn __str__(&self) -> String { + format!("({}, '{}')", self.0, self.1) + } + } + + /// A reference from outside the current scope of execution. + /// + /// This could be an input or the body of a node. + #[pyclass(eq, frozen)] + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] + pub struct ExteriorRef(pub PortID); + + impl std::fmt::Display for ExteriorRef { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "ExteriorRef({})", self.0)?; + Ok(()) + } + } + + #[pymethods] + impl ExteriorRef { + #[new] + pub fn new(port_id: PortID) -> Self { + Self(port_id) + } + + #[getter] + pub fn port_id(&self) -> PortID { + self.0.clone() + } + + pub fn __iter__<'py>(&self, py: Python<'py>) -> PyResult> { + PyIterator::from_object(&(ExteriorRef(self.0.clone())).into_pyobject(py)?.into_any()) + } + + pub fn __repr__(&self) -> String { + format!("{}", self) + } + + pub fn __str__(&self) -> String { + self.0.to_string() + } + } + + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, FromPyObject, IntoPyObject)] + pub enum ExteriorOrValueRef { + Exterior(ExteriorRef), + Value(ValueRef), + } + + impl std::fmt::Display for ExteriorOrValueRef { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Exterior(x) => write!(f, "{}", x), + Self::Value(x) => write!(f, "{}", x), + } + } + } +} diff --git a/tierkreis_core/rust/lib.rs b/tierkreis_core/rust/lib.rs new file mode 100644 index 000000000..f5d229abe --- /dev/null +++ b/tierkreis_core/rust/lib.rs @@ -0,0 +1,30 @@ +pub mod graph; +pub mod identifiers; +pub mod location; +pub mod value; + +use pyo3::prelude::*; + +#[pymodule] +pub mod _tierkreis_core { + use super::*; + + /// Hack: workaround for https://github.com/PyO3/pyo3/issues/759 + #[pymodule_init] + fn init(m: &Bound<'_, PyModule>) -> PyResult<()> { + Python::attach(|py| { + py.import("sys")? + .getattr("modules")? + .set_item("tierkreis_core._tierkreis_core", m) + }) + } + + #[pymodule_export] + use super::graph::graph; + #[pymodule_export] + use super::identifiers::identifiers; + #[pymodule_export] + use super::location::location; + #[pymodule_export] + use super::value::value; +} diff --git a/tierkreis_core/rust/location.rs b/tierkreis_core/rust/location.rs new file mode 100644 index 000000000..154d1c36a --- /dev/null +++ b/tierkreis_core/rust/location.rs @@ -0,0 +1,374 @@ +use pyo3::{exceptions::PyValueError, prelude::*, types::PyIterator}; +use serde::{Deserialize, Serialize}; + +#[pymodule(submodule)] +pub mod location { + use std::collections::{HashMap, VecDeque}; + + use pyo3::types::{IntoPyDict, PyType}; + + use crate::identifiers::identifiers::{ExteriorOrValueRef, NodeIndex}; + + use super::*; + + /// Hack: workaround for https://github.com/PyO3/pyo3/issues/759 + #[pymodule_init] + fn init(m: &Bound<'_, PyModule>) -> PyResult<()> { + Python::attach(|py| { + py.import("sys")? + .getattr("modules")? + .set_item("tierkreis_core._tierkreis_core.location", m) + }) + } + + #[pyclass(eq, frozen, hash)] + #[derive(Clone, PartialEq, Serialize, Deserialize, Hash, Debug)] + pub enum NodeStep { + // The root step, should always be the first step if + // it is present at all. + Root {}, + // A single Node, such as a function, cosnt or an eval. + Node { node_index: NodeIndex }, + Loop { loop_index: usize }, + Map { map_index: usize }, + // A backtraking step to a location with exterior refs + Exterior {}, + } + + impl std::fmt::Display for NodeStep { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + NodeStep::Root {} => write!(f, "-")?, + NodeStep::Node { node_index } => write!(f, "N{}", node_index.0)?, + NodeStep::Loop { loop_index } => write!(f, "L{}", loop_index)?, + NodeStep::Map { map_index } => write!(f, "M{}", map_index)?, + NodeStep::Exterior {} => write!(f, "E")?, + } + Ok(()) + } + } + + // These methods are mostly for tests, users should prefer + // using the methods on Loc instead. + #[pymethods] + impl NodeStep { + #[new] + pub fn new(steps: &str) -> PyResult { + match (steps.get(0..1), steps.get(1..)) { + (Some("-"), Some("")) => Ok(NodeStep::Root {}), + (Some("N"), Some(idx_str)) => Ok(NodeStep::Node { + node_index: NodeIndex(idx_str.parse()?), + }), + (Some("L"), Some(idx_str)) => Ok(NodeStep::Loop { + loop_index: idx_str.parse()?, + }), + (Some("M"), Some(idx_str)) => Ok(NodeStep::Map { + map_index: idx_str.parse()?, + }), + (Some("E"), Some("")) => Ok(NodeStep::Exterior {}), + (step, index) => Err(PyValueError::new_err(format!( + "Could not parse Loc: {} with step {:?} and index {:?}", + steps, step, index + ))), + } + } + + fn is_root(&self) -> bool { + self == &NodeStep::Root {} + } + + pub fn __iter__<'py>(&self, py: Python<'py>) -> PyResult> { + match self { + Self::Root {} => PyIterator::from_object(&"-".into_pyobject(py)?.into_any()), + Self::Node { node_index } => { + PyIterator::from_object(&("N", *node_index).into_pyobject(py)?.into_any()) + } + Self::Loop { loop_index } => { + PyIterator::from_object(&("L", loop_index).into_pyobject(py)?.into_any()) + } + Self::Map { map_index } => { + PyIterator::from_object(&("M", map_index).into_pyobject(py)?.into_any()) + } + Self::Exterior {} => PyIterator::from_object(&"E".into_pyobject(py)?.into_any()), + } + } + + pub fn __repr__(&self) -> String { + format!("NodeStep('{}')", self) + } + + pub fn __str__(&self) -> String { + format!("{}", self) + } + } + + /// A location where a Value is stored, expressed as + /// a sequence of NodeStep. + #[pyclass(eq, frozen, hash)] + #[derive(Clone, PartialEq, Serialize, Deserialize, Hash, Debug)] + pub struct Loc { + steps: VecDeque, + } + + impl std::fmt::Display for Loc { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.steps + .front() + .map(|first_step| write!(f, "{}", first_step)) + .transpose()?; + for step in self.steps.iter().skip(1) { + write!(f, ".{}", step)?; + } + Ok(()) + } + } + + #[derive(FromPyObject)] + pub enum NodeIndexOrPositiveInt { + NodeIndex(NodeIndex), + Int(usize), + } + + #[derive(FromPyObject)] + pub enum LocOrString { + Loc(Loc), + String(String), + } + + #[pymethods] + impl Loc { + #[new] + #[pyo3(signature = (k: "str" = "-"))] + pub fn new(k: &str) -> PyResult { + let parts = k.split_terminator("."); + let mut steps = VecDeque::new(); + for part in parts { + steps.push_back(NodeStep::new(part)?); + } + Ok(Self { steps }) + } + + pub fn extend_from_ref(&self, value_ref: ExteriorOrValueRef) -> Self { + let mut steps = self.steps.clone(); + match value_ref { + ExteriorOrValueRef::Exterior(_) => steps.push_back(NodeStep::Exterior {}), + ExteriorOrValueRef::Value(value_ref) => steps.push_back(NodeStep::Node { + node_index: value_ref.node_index(), + }), + }; + Loc { steps } + } + + pub fn exterior(&self) -> Self { + let mut steps = self.steps.clone(); + steps.push_back(NodeStep::Exterior {}); + Loc { steps } + } + + #[pyo3(name = "N")] + pub fn append_node(&self, node_index: NodeIndexOrPositiveInt) -> Self { + let mut steps = self.steps.clone(); + let node_index = match node_index { + NodeIndexOrPositiveInt::NodeIndex(node_index) => node_index, + NodeIndexOrPositiveInt::Int(i) => NodeIndex(i), + }; + steps.push_back(NodeStep::Node { node_index }); + Loc { steps } + } + + #[pyo3(name = "L")] + pub fn append_loop(&self, loop_index: usize) -> Self { + let mut steps = self.steps.clone(); + steps.push_back(NodeStep::Loop { loop_index }); + Loc { steps } + } + + #[pyo3(name = "M")] + pub fn append_map(&self, map_index: usize) -> Self { + let mut steps = self.steps.clone(); + steps.push_back(NodeStep::Map { map_index }); + Loc { steps } + } + + #[staticmethod] + #[pyo3(signature = ( + steps: "typing.Sequence[NodeStep]", + ))] + pub fn from_steps(steps: Vec) -> Self { + Self { + steps: steps.into(), + } + } + + #[pyo3(signature = () -> "list[Loc]")] + pub fn steps(&self) -> Vec { + self.steps.iter().cloned().collect() + } + + /// `parent` will get the last step in Loc, unless the last + /// step is a `Loop` step, in which case it will get the + /// previous iteration of the `Loop` step by subtracting 1. + #[pyo3(signature = () -> "Loc | None")] + pub fn parent(&self) -> Option { + self.steps.back().map(|last_step| match last_step { + NodeStep::Root {} => Loc { + steps: VecDeque::new(), + }, + NodeStep::Node { .. } | NodeStep::Map { .. } | NodeStep::Exterior { .. } => { + let mut steps = self.steps.clone(); + steps.pop_back(); + + Loc { steps } + } + NodeStep::Loop { loop_index } if *loop_index == 0 => { + let mut steps = self.steps.clone(); + steps.pop_back(); + + Loc { steps } + } + NodeStep::Loop { loop_index } => { + let mut steps = self.steps.clone(); + steps.pop_back(); + steps.push_back(NodeStep::Loop { + loop_index: loop_index - 1, + }); + + Loc { steps } + } + }) + } + + // TODO: this needs tests + pub fn startswith(&self, other: &Loc) -> bool { + self.steps + .iter() + .zip(other.steps.iter()) + .all(|(x, y)| x == y) + } + + // Returns true if he Loc only contains a root node. + pub fn is_root_loc(&self) -> bool { + self.steps.len() == 1 && self.steps.front() == Some(&NodeStep::Root {}) + } + + // Returns true if the last step in the sequence is an exterior step. + pub fn last_step_exterior(&self) -> bool { + self.steps.back() == Some(&NodeStep::Exterior {}) + } + + // Returns the index of the step if it is a loop or map step. + #[pyo3(signature = () -> "int | None")] + pub fn peek_index(&self) -> Option { + self.steps.back().and_then(|step| match step { + NodeStep::Root {} => None, + NodeStep::Node { .. } => None, + NodeStep::Loop { loop_index } => Some(*loop_index), + NodeStep::Map { map_index } => Some(*map_index), + NodeStep::Exterior {} => None, + }) + } + + pub fn partial_locs(&self) -> Vec { + let mut partial_locs = Vec::new(); + let mut intermediate = VecDeque::new(); + for step in &self.steps { + intermediate.push_back(step.clone()); + partial_locs.push(Loc { + steps: intermediate.clone(), + }); + } + partial_locs + } + + #[pyo3(signature = () -> "tuple[NodeStep, Loc]")] + pub fn pop_first(&self) -> PyResult<(NodeStep, Loc)> { + if self.steps.len() == 1 && self.steps.front() == Some(&NodeStep::Root {}) { + return Ok(( + NodeStep::Root {}, + Loc { + steps: VecDeque::new(), + }, + )); + } + if self.steps.len() < 2 { + return Err(PyValueError::new_err("Cannot pop from empty Loc")); + } + let mut steps = self.steps.clone(); + // We never pop the root step and instead pop the next one along. + let first = steps.remove(1); + if first == Some(NodeStep::Root {}) { + return Err(PyValueError::new_err("Malformed Loc")); + } + Ok((first.unwrap(), Loc { steps })) + } + + #[pyo3(signature = () -> "tuple[NodeStep, Loc]")] + pub fn pop_last(&self) -> PyResult<(NodeStep, Loc)> { + if self.steps.len() == 1 && self.steps.front() == Some(&NodeStep::Root {}) { + return Ok(( + NodeStep::Root {}, + Loc { + steps: VecDeque::new(), + }, + )); + } + if self.steps.len() < 2 { + return Err(PyValueError::new_err("Cannot pop from empty Loc")); + } + let mut steps = self.steps.clone(); + let first = steps.pop_back(); + if first == Some(NodeStep::Root {}) { + return Err(PyValueError::new_err("Malformed Loc")); + } + Ok((first.unwrap(), Loc { steps })) + } + + pub fn __fspath__(&self) -> String { + format!("{}", self) + } + + pub fn __repr__(&self) -> String { + format!("Loc('{}')", self) + } + + pub fn __str__(&self) -> String { + format!("{}", self) + } + + // pydantic compatibility + #[classmethod] + pub fn __get_pydantic_core_schema__( + cls: &Bound<'_, PyType>, + _source: &Bound<'_, PyType>, + _handler: &Bound<'_, PyAny>, + ) -> PyResult> { + let pydantic = PyModule::import(cls.py(), "pydantic_core")?; + let core_schema = pydantic.getattr("core_schema")?; + + let validate_fn = cls.getattr("_validate")?; + // Used to validate the output of the schema + let any_schema = core_schema.call_method0("any_schema")?; + + let serialization = core_schema.call_method0("to_string_ser_schema")?; + + let mut validator_kwargs = HashMap::new(); + validator_kwargs.insert("serialization", serialization); + + let schema = core_schema.call_method( + "no_info_before_validator_function", + (validate_fn, any_schema), + Some(&validator_kwargs.into_py_dict(cls.py())?), + )?; + + Ok(schema.unbind()) + } + + #[staticmethod] + pub fn _validate(value: LocOrString) -> PyResult { + match value { + LocOrString::Loc(loc) => Ok(loc), + LocOrString::String(s) => Loc::new(&s), + } + } + } +} diff --git a/tierkreis_core/rust/value.rs b/tierkreis_core/rust/value.rs new file mode 100644 index 000000000..6b58f4402 --- /dev/null +++ b/tierkreis_core/rust/value.rs @@ -0,0 +1,33 @@ +use std::collections::HashMap; + +use pyo3::prelude::*; +use serde::{Deserialize, Serialize}; + +#[pymodule(submodule)] +pub mod value { + use crate::graph::graph::GraphData; + + use super::*; + + /// Hack: workaround for https://github.com/PyO3/pyo3/issues/759 + #[pymodule_init] + fn init(m: &Bound<'_, PyModule>) -> PyResult<()> { + Python::attach(|py| { + py.import("sys")? + .getattr("modules")? + .set_item("tierkreis_core._tierkreis_core.value", m) + }) + } + + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, FromPyObject, IntoPyObject)] + pub enum Value { + Bool(bool), + Int(i64), + Float(f64), + Str(String), + List(Vec), + Map(HashMap), + Bytes(Vec), + Graph(GraphData), + } +} diff --git a/tierkreis_visualization/tierkreis_visualization/app.py b/tierkreis_visualization/tierkreis_visualization/app.py index aa9a3bea6..b2043bf54 100644 --- a/tierkreis_visualization/tierkreis_visualization/app.py +++ b/tierkreis_visualization/tierkreis_visualization/app.py @@ -1,12 +1,13 @@ import signal from sys import argv -from tierkreis.controller.data.graph import GraphData from tierkreis_visualization.app_config import ( App, StorageType, graph_data_lifespan, dev_lifespan, ) + +from tierkreis_core import GraphData from tierkreis_visualization.config import CONFIG from tierkreis_visualization.storage import ( file_storage_fn, diff --git a/tierkreis_visualization/tierkreis_visualization/data/eval.py b/tierkreis_visualization/tierkreis_visualization/data/eval.py index 4bf12b432..03c3151e4 100644 --- a/tierkreis_visualization/tierkreis_visualization/data/eval.py +++ b/tierkreis_visualization/tierkreis_visualization/data/eval.py @@ -1,12 +1,9 @@ import json -from typing import Optional, assert_never +from typing import Optional -from tierkreis.controller.data.core import NodeIndex -from tierkreis.controller.data.location import Loc -from tierkreis.controller.data.graph import GraphData, IfElse, NodeDef from tierkreis.controller.data.types import ptype_from_bytes -from tierkreis.controller.storage.adjacency import in_edges from tierkreis.controller.storage.protocol import ControllerStorage +from tierkreis_core import GraphData, Loc, NodeDef, NodeDescription from tierkreis.exceptions import TierkreisError from tierkreis_visualization.data.models import PyNode, NodeStatus, PyEdge @@ -15,7 +12,7 @@ def node_status( - is_finished: bool, definition: Optional[NodeDef], has_error: bool = False + is_finished: bool, definition: Optional[NodeDescription], has_error: bool = False ) -> NodeStatus: if is_finished: return "Finished" @@ -35,12 +32,14 @@ def check_error(node_location: Loc, errored_nodes: list[Loc]) -> bool: def add_conditional_edges( storage: ControllerStorage, loc: Loc, - i: NodeIndex, - node: IfElse, + i: int, + node: NodeDef.IfElse | NodeDef.EagerIfElse, py_edges: list[PyEdge], ): try: - pred = json.loads(storage.read_output(loc.N(node.pred[0]), node.pred[1])) + pred = json.loads( + storage.read_output(loc.N(node.pred.node_index), node.pred.port_id) + ) except (FileNotFoundError, TierkreisError): pred = None @@ -65,7 +64,7 @@ def add_conditional_edges( def get_eval_node( storage: ControllerStorage, node_location: Loc, errored_nodes: list[Loc] ) -> PyGraph: - thunk = storage.read_output(node_location.N(-1), "body") + thunk = storage.read_output(node_location.exterior(), "body") graph = ptype_from_bytes(thunk, GraphData) pynodes: list[PyNode] = [] @@ -76,7 +75,7 @@ def get_eval_node( is_finished = storage.is_node_finished(new_location) has_error = check_error(new_location, errored_nodes) try: - definition = storage.read_node_def(new_location) + definition = storage.read_node_description(new_location) except (FileNotFoundError, TierkreisError): definition = None @@ -84,45 +83,63 @@ def get_eval_node( started_time = storage.read_started_time(new_location) or "" finished_time = storage.read_finished_time(new_location) or "" value: str | None = None - match node.type: - case "function": - name = node.function_name - case "ifelse": - name = node.type + node_type: str + match node: + case NodeDef.Func(): + name = node.name + node_type = "function" + case NodeDef.IfElse(): + name = "ifelse" + node_type = "ifelse" add_conditional_edges(storage, node_location, i, node, py_edges) - case "map" | "eval" | "loop" | "eifelse": - name = node.type - case "const": - name = node.type + case NodeDef.Map(): + name = "map" + node_type = "map" + case NodeDef.Eval(): + name = "eval" + node_type = "eval" + case NodeDef.Loop(): + name = "loop" + node_type = "loop" + case NodeDef.EagerIfElse(): + name = "eifelse" + node_type = "eifelse" + case NodeDef.Const(): + name = "const" + node_type = "const" value = outputs_from_loc(storage, node_location.N(i), "value") - case "output": - name = node.type + case NodeDef.Output(): + name = "output" + node_type = "output" if len(node.inputs) == 1: - (idx, p) = next(iter(node.inputs.values())) + ref = next(iter(node.inputs.values())) try: - value = outputs_from_loc(storage, node_location.N(idx), p) + value = outputs_from_loc( + storage, node_location.extend_from_ref(ref), ref.port_id + ) except (FileNotFoundError, TierkreisError): value = None - case "input": - name = node.type + case NodeDef.Input(): + name = "input" + node_type = "input" value = node.name case _: - assert_never(node) + raise ValueError(f"Unhandled NodeDef of type: {type(node)}") pynode = PyNode( id=new_location, status=status, function_name=name, node_location=new_location, - node_type=node.type, + node_type=node_type, value=value, started_time=started_time, finished_time=finished_time, - outputs=list(node.outputs), + outputs=list(definition.outputs) if definition is not None else [], ) pynodes.append(pynode) - for p0, (idx, p1) in in_edges(node).items(): + for p0, (idx, p1) in node.in_edges.items(): value: str | None = None try: diff --git a/tierkreis_visualization/tierkreis_visualization/data/graph.py b/tierkreis_visualization/tierkreis_visualization/data/graph.py index db0f49820..75fdbd725 100644 --- a/tierkreis_visualization/tierkreis_visualization/data/graph.py +++ b/tierkreis_visualization/tierkreis_visualization/data/graph.py @@ -1,7 +1,8 @@ -from typing import assert_never from fastapi import HTTPException from tierkreis.controller.data.location import Loc from tierkreis.controller.storage.protocol import ControllerStorage +from tierkreis_core import NodeDef + from tierkreis_visualization.data.eval import get_eval_node from tierkreis_visualization.data.loop import get_loop_node from tierkreis_visualization.data.map import get_map_node @@ -21,27 +22,36 @@ def get_node_data(storage: ControllerStorage, loc: Loc) -> PyGraph: errored_nodes = get_errored_nodes(storage) try: - node = storage.read_node_def(loc) + description = storage.read_node_description(loc) except FileNotFoundError: raise HTTPException(404, detail="Node definition not found.") - match node.type: - case "eval": + match description.definition: + case NodeDef.Eval(): data = get_eval_node(storage, loc, errored_nodes) return PyGraph(nodes=data.nodes, edges=data.edges) - case "loop": + case NodeDef.Loop(): data = get_loop_node(storage, loc, errored_nodes) return PyGraph(nodes=data.nodes, edges=data.edges) - case "map": - data = get_map_node(storage, loc, node, errored_nodes) + case NodeDef.Map(): + data = get_map_node(storage, loc, description.definition, errored_nodes) return PyGraph(nodes=data.nodes, edges=data.edges) - case "function" | "const" | "ifelse" | "eifelse" | "input" | "output": + case ( + NodeDef.Func + | NodeDef.Const + | NodeDef.IfElse + | NodeDef.EagerIfElse + | NodeDef.Input + | NodeDef.Output + ): raise HTTPException( 400, detail="Only eval, loop and map nodes return a graph." ) case _: - assert_never(node) + raise ValueError( + f"Unhandled NodeDef of type: {type(description.definition)}" + ) diff --git a/tierkreis_visualization/tierkreis_visualization/data/loop.py b/tierkreis_visualization/tierkreis_visualization/data/loop.py index e844dfa7e..59fdb6102 100644 --- a/tierkreis_visualization/tierkreis_visualization/data/loop.py +++ b/tierkreis_visualization/tierkreis_visualization/data/loop.py @@ -20,7 +20,7 @@ def get_loop_node( while storage.is_node_started(node_location.L(i + 1)): i += 1 new_location = node_location.L(i) - nodedef = storage.read_node_def(new_location) + description = storage.read_node_description(new_location) nodes = [ PyNode( @@ -31,7 +31,7 @@ def get_loop_node( node_type="eval", started_time=storage.read_started_time(node_location.L(n)) or "", finished_time=storage.read_finished_time(node_location.L(n)) or "", - outputs=list(nodedef.outputs), + outputs=list(description.outputs), ) for n in range(i) ] @@ -51,11 +51,12 @@ def get_loop_node( node_type="eval", started_time=storage.read_started_time(new_location) or "", finished_time=storage.read_finished_time(new_location) or "", - outputs=list(nodedef.outputs), + outputs=list(description.outputs), ) ) edges = [] - for port_name in storage.read_node_def(node_location.L(0)).outputs: + description = storage.read_node_description(node_location.L(0)) + for port_name in description.outputs: edges.extend( [ PyEdge( diff --git a/tierkreis_visualization/tierkreis_visualization/data/map.py b/tierkreis_visualization/tierkreis_visualization/data/map.py index 79bc31503..b5fb2f451 100644 --- a/tierkreis_visualization/tierkreis_visualization/data/map.py +++ b/tierkreis_visualization/tierkreis_visualization/data/map.py @@ -1,11 +1,10 @@ from pydantic import BaseModel -from tierkreis.controller.data.location import Loc from tierkreis.controller.storage.adjacency import outputs_iter from tierkreis.controller.storage.protocol import ControllerStorage -from tierkreis.controller.data.graph import Map from tierkreis.exceptions import TierkreisError from tierkreis_visualization.data.eval import check_error from tierkreis_visualization.data.models import PyEdge, PyNode +from tierkreis_core import Loc, NodeDef class MapNodeData(BaseModel): @@ -14,15 +13,15 @@ class MapNodeData(BaseModel): def get_map_node( - storage: ControllerStorage, loc: Loc, map: Map, errored_nodes: list[Loc] + storage: ControllerStorage, loc: Loc, map: NodeDef.Map, errored_nodes: list[Loc] ) -> MapNodeData: parent = loc.parent() if parent is None: raise TierkreisError("MAP node must have parent.") - first_ref = next(x for x in map.inputs.values() if x[1] == "*") - map_eles = outputs_iter(storage, parent.N(first_ref[0])) - nodedef = storage.read_node_def(loc.M(0)) + first_ref = next(x for x in map.inputs.values() if x.port_id == "*") + map_eles = outputs_iter(storage, parent.extend_from_ref(first_ref)) + description = storage.read_node_description(loc.M(0)) nodes: list[PyNode] = [] for i, ele in map_eles: node = PyNode( @@ -33,7 +32,7 @@ def get_map_node( node_type="eval", started_time=storage.read_started_time(loc.M(i)) or "", finished_time=storage.read_finished_time(loc.M(i)) or "", - outputs=list(nodedef.outputs), + outputs=list(description.outputs), ) if check_error(loc.M(i), errored_nodes): node.status = "Error" diff --git a/tierkreis_visualization/tierkreis_visualization/data/models.py b/tierkreis_visualization/tierkreis_visualization/data/models.py index 3b1b86aca..e55d3f031 100644 --- a/tierkreis_visualization/tierkreis_visualization/data/models.py +++ b/tierkreis_visualization/tierkreis_visualization/data/models.py @@ -1,6 +1,6 @@ from typing import Literal from pydantic import BaseModel -from tierkreis.controller.data.location import Loc +from tierkreis_core import Loc NodeStatus = Literal["Not started", "Started", "Error", "Finished"] NodeType = Literal[ @@ -13,7 +13,7 @@ class PyNode(BaseModel): status: NodeStatus function_name: str node_type: NodeType - node_location: str = "" + node_location: Loc = Loc("") outputs: list[str] value: str | None = None started_time: str diff --git a/tierkreis_visualization/tierkreis_visualization/routers/models.py b/tierkreis_visualization/tierkreis_visualization/routers/models.py index 977b4ce7e..8aecc61f8 100644 --- a/tierkreis_visualization/tierkreis_visualization/routers/models.py +++ b/tierkreis_visualization/tierkreis_visualization/routers/models.py @@ -1,5 +1,5 @@ from pydantic import BaseModel -from tierkreis.controller.data.location import Loc +from tierkreis_core import Loc from tierkreis_visualization.data.models import PyNode, PyEdge diff --git a/tierkreis_visualization/tierkreis_visualization/routers/navigation.py b/tierkreis_visualization/tierkreis_visualization/routers/navigation.py new file mode 100644 index 000000000..bde253b5b --- /dev/null +++ b/tierkreis_visualization/tierkreis_visualization/routers/navigation.py @@ -0,0 +1,19 @@ +from uuid import UUID + +from tierkreis_core import Loc + + +def breadcrumb_links(crumbs: list[str]) -> list[tuple[str, str]]: + url_path = "" + links: list[tuple[str, str]] = [] + for crumb in crumbs: + url_path = url_path + crumb + links.append((crumb, url_path)) + + return links + + +def breadcrumbs(workflow_id: UUID, node_location: Loc) -> list[tuple[str, str]]: + node_location_strs: list[str] = [str(x) for x in node_location.steps()] + static_links = ["/workflows", f"/{workflow_id}/nodes/-"] + return breadcrumb_links(static_links + node_location_strs[1:]) diff --git a/tierkreis_visualization/tierkreis_visualization/routers/workflows.py b/tierkreis_visualization/tierkreis_visualization/routers/workflows.py index 14115ed19..607dd7b22 100644 --- a/tierkreis_visualization/tierkreis_visualization/routers/workflows.py +++ b/tierkreis_visualization/tierkreis_visualization/routers/workflows.py @@ -50,7 +50,7 @@ async def handle_websocket( if not path.parts: continue loc = path.parts[0] - if loc.startswith(node_location): + if loc.startswith(str(node_location)): relevant_changes.add(loc) if relevant_changes: diff --git a/tierkreis_visualization/tierkreis_visualization/storage.py b/tierkreis_visualization/tierkreis_visualization/storage.py index 647f871b5..fcbcd185b 100644 --- a/tierkreis_visualization/tierkreis_visualization/storage.py +++ b/tierkreis_visualization/tierkreis_visualization/storage.py @@ -5,10 +5,10 @@ from typing import Callable from uuid import UUID -from tierkreis.controller.data.graph import GraphData from tierkreis.controller.storage.filestorage import ControllerFileStorage from tierkreis.controller.storage.graphdata import GraphDataStorage from tierkreis.controller.storage.protocol import ControllerStorage +from tierkreis_core import GraphData def file_storage_fn(tkr_dir: Path) -> Callable[[UUID], ControllerStorage]: diff --git a/tierkreis_visualization/tierkreis_visualization/visualize_graph.py b/tierkreis_visualization/tierkreis_visualization/visualize_graph.py index d25936f8c..d1fd72874 100644 --- a/tierkreis_visualization/tierkreis_visualization/visualize_graph.py +++ b/tierkreis_visualization/tierkreis_visualization/visualize_graph.py @@ -1,7 +1,7 @@ import uvicorn from tierkreis.builder import GraphBuilder -from tierkreis.controller.data.graph import GraphData +from tierkreis_core import GraphData from tierkreis_visualization.app import app_from_graph_data diff --git a/uv.lock b/uv.lock index 92e89a2c9..2bc287d48 100644 --- a/uv.lock +++ b/uv.lock @@ -5,6 +5,7 @@ requires-python = ">=3.12" [manifest] members = [ "tierkreis", + "tierkreis-core", "tierkreis-visualization", "tkr-aer-worker", "tkr-ibmq-worker", @@ -2901,6 +2902,7 @@ version = "2.0.9" source = { editable = "tierkreis" } dependencies = [ { name = "pydantic" }, + { name = "tierkreis-core" }, ] [package.dev-dependencies] @@ -2913,7 +2915,10 @@ dev = [ ] [package.metadata] -requires-dist = [{ name = "pydantic", specifier = "~=2.5" }] +requires-dist = [ + { name = "pydantic", specifier = "~=2.5" }, + { name = "tierkreis-core", editable = "tierkreis_core" }, +] [package.metadata.requires-dev] build = [{ name = "build", extras = ["uv"] }] @@ -2922,6 +2927,10 @@ dev = [ { name = "pytest" }, ] +[[package]] +name = "tierkreis-core" +source = { editable = "tierkreis_core" } + [[package]] name = "tierkreis-visualization" version = "0.1.6" From e8f14154892d3c16507ceaacc65dce7fb101df84 Mon Sep 17 00:00:00 2001 From: John Children Date: Fri, 9 Jan 2026 10:41:39 +0000 Subject: [PATCH 2/6] chore: Try to avoid .inputs --- tierkreis/tierkreis/cli/run_workflow.py | 2 +- tierkreis/tierkreis/controller/start.py | 5 ++-- .../tierkreis/controller/storage/walk.py | 4 +-- tierkreis/tierkreis/storage.py | 2 +- .../tierkreis_core/_tierkreis_core/graph.pyi | 12 ++++---- .../rust/bin/tierkreis-core-stubs-gen.rs | 12 ++------ tierkreis_core/rust/graph.rs | 28 ++----------------- .../tierkreis_visualization/data/eval.py | 5 ++-- 8 files changed, 22 insertions(+), 48 deletions(-) diff --git a/tierkreis/tierkreis/cli/run_workflow.py b/tierkreis/tierkreis/cli/run_workflow.py index 1abca0c51..9881a8dc3 100644 --- a/tierkreis/tierkreis/cli/run_workflow.py +++ b/tierkreis/tierkreis/cli/run_workflow.py @@ -56,6 +56,6 @@ def run_workflow( polling_interval_seconds, ) if print_output and (output_idx := graph.output_idx()): - all_outputs = graph.get_nodedef(output_idx).inputs + all_outputs = graph.get_nodedef(output_idx).in_edges for output in all_outputs: print(f"{output}: {storage.read_output(Loc(), output)}") diff --git a/tierkreis/tierkreis/controller/start.py b/tierkreis/tierkreis/controller/start.py index 4285d0bda..401f722f0 100644 --- a/tierkreis/tierkreis/controller/start.py +++ b/tierkreis/tierkreis/controller/start.py @@ -86,7 +86,8 @@ def start( raise TierkreisError(f"{type(node)} node must have parent Loc.") ins = { - k: (parent.extend_from_ref(ref), ref.port_id) for k, ref in node.inputs.items() + k: (parent.extend_from_ref(ref), ref.port_id) + for k, ref in node.in_edges.items() } logger.debug(f"start {node_location} {node} {ins}") @@ -129,7 +130,7 @@ def start( body_loc = parent.extend_from_ref(node.body) message = storage.read_output(body_loc, node.body.port_id) g = ptype_from_bytes(message, GraphData) - ins["body"] = (body_loc, node.body.port_id) + # ins["body"] = (body_loc, node.body.port_id) ins.update(g.fixed_inputs) pipe_inputs_to_output_location(storage, node_location.exterior(), ins) diff --git a/tierkreis/tierkreis/controller/storage/walk.py b/tierkreis/tierkreis/controller/storage/walk.py index 1a71fd29a..dff9585cb 100644 --- a/tierkreis/tierkreis/controller/storage/walk.py +++ b/tierkreis/tierkreis/controller/storage/walk.py @@ -138,7 +138,7 @@ def walk_loop( return walk_node(storage, new_location, output_idx, g) # The outputs from the previous iteration - body_outputs = g.get_nodedef(output_idx).inputs + body_outputs = g.get_nodedef(output_idx).in_edges if body_outputs is None: raise ValueError("Loop body has no outputs.") @@ -202,7 +202,7 @@ def walk_map( if len(unfinished) > 0: return result - map_outputs = g.get_nodedef(output_idx).inputs + map_outputs = g.get_nodedef(output_idx).in_edges for i, j in map_eles: for output in map_outputs.keys(): storage.link_outputs(loc, f"{output}-{j}", loc.M(i), output) diff --git a/tierkreis/tierkreis/storage.py b/tierkreis/tierkreis/storage.py index b0c20aea4..83f377a14 100644 --- a/tierkreis/tierkreis/storage.py +++ b/tierkreis/tierkreis/storage.py @@ -24,7 +24,7 @@ def read_outputs( if output_idx is None: raise ValueError("Cannot read outputs of a graph with no Output node.") - out_ports = list(g.get_nodedef(output_idx).inputs.keys()) + out_ports = list(g.get_nodedef(output_idx).in_edges.keys()) if len(out_ports) == 1 and "value" in out_ports: return ptype_from_bytes(storage.read_output(Loc(), "value")) return {k: ptype_from_bytes(storage.read_output(Loc(), k)) for k in out_ports} diff --git a/tierkreis_core/python/tierkreis_core/_tierkreis_core/graph.pyi b/tierkreis_core/python/tierkreis_core/_tierkreis_core/graph.pyi index 6c13d8058..f060b2259 100644 --- a/tierkreis_core/python/tierkreis_core/_tierkreis_core/graph.pyi +++ b/tierkreis_core/python/tierkreis_core/_tierkreis_core/graph.pyi @@ -108,12 +108,12 @@ class NodeDef: ) -> dict[ tierkreis_core.aliases.PortID, identifiers.ValueRef | identifiers.ExteriorRef ]: ... - @property - def inputs( - self, / - ) -> dict[ - tierkreis_core.aliases.PortID, identifiers.ValueRef | identifiers.ExteriorRef - ]: ... + # @property + # def inputs( + # self, / + # ) -> dict[ + # tierkreis_core.aliases.PortID, identifiers.ValueRef | identifiers.ExteriorRef + # ]: ... def model_dump_json(self, /) -> str: ... @classmethod def model_load_json(cls, /, s: str) -> graph.NodeDef: ... diff --git a/tierkreis_core/rust/bin/tierkreis-core-stubs-gen.rs b/tierkreis_core/rust/bin/tierkreis-core-stubs-gen.rs index facc907f0..4e9a4b91c 100644 --- a/tierkreis_core/rust/bin/tierkreis-core-stubs-gen.rs +++ b/tierkreis_core/rust/bin/tierkreis-core-stubs-gen.rs @@ -67,25 +67,19 @@ pub fn main() { // Extra patches for erroneous typing.Any on properties. - // // Patch nodes on GraphData + // Patch nodes on GraphData patched_stub = patched_stub.replace( "def nodes(self, /) -> typing.Any", "def nodes(self, /) -> list[NodeDef]", ); - // // Patch inputs on NodeDef - patched_stub = patched_stub.replace( - "def inputs(self, /) -> typing.Any", - "def inputs(self, /) -> dict[tierkreis_core.aliases.PortID, identifiers.ValueRef | identifiers.ExteriorRef]", - ); - - // // Patch in_edges on NodeDef + // Patch in_edges on NodeDef patched_stub = patched_stub.replace( "def in_edges(self, /) -> typing.Any", "def in_edges(self, /) -> dict[tierkreis_core.aliases.PortID, identifiers.ValueRef | identifiers.ExteriorRef]", ); - // // Patch outer_graph on NodeDescription + // Patch outer_graph on NodeDescription patched_stub = patched_stub.replace( "def outer_graph(self, /) -> typing.Any", "def outer_graph(self, /) -> graph.GraphData | None", diff --git a/tierkreis_core/rust/graph.rs b/tierkreis_core/rust/graph.rs index f836b7b5d..b4ff4b6a5 100644 --- a/tierkreis_core/rust/graph.rs +++ b/tierkreis_core/rust/graph.rs @@ -120,31 +120,9 @@ pub mod graph { #[pymethods] impl NodeDef { - /// Get the `inputs` attribute for a Node. This is different to - /// `in_edges` as it only considers the edges under the `inputs` - /// attribute and not things like the body of Eval nodes which - /// is considered a different kind of edge. - #[getter] - pub fn inputs(&self) -> IndexMap { - match self { - Self::Eval { inputs, .. } => inputs.clone(), - Self::Loop { inputs, .. } => inputs.clone(), - Self::Map { inputs, .. } => inputs.clone(), - Self::Func { inputs, .. } => inputs.clone(), - Self::Const { .. } => IndexMap::new(), - Self::IfElse { .. } => IndexMap::new(), - Self::EagerIfElse { .. } => IndexMap::new(), - Self::Input { .. } => IndexMap::new(), - Self::Output { inputs, .. } => inputs - .iter() - .map(|x| (x.0.clone(), ExteriorOrValueRef::Value(x.1.clone()))) - .collect(), - } - } - - /// Get the `in_edges` attribute for a Node. This is different to - /// `inputs` as it also considers the edges connected to the body - /// port of Eval nodes which does not come under the `inputs` attribute. + /// Get the `in_edges` attribute for a Node. This is dictionary + /// also considers the edges connected to the body port of Eval + /// nodes which does not come under the `inputs` attribute. #[getter] pub fn in_edges(&self) -> IndexMap { match self { diff --git a/tierkreis_visualization/tierkreis_visualization/data/eval.py b/tierkreis_visualization/tierkreis_visualization/data/eval.py index 03c3151e4..a4cdbbd2c 100644 --- a/tierkreis_visualization/tierkreis_visualization/data/eval.py +++ b/tierkreis_visualization/tierkreis_visualization/data/eval.py @@ -111,8 +111,9 @@ def get_eval_node( case NodeDef.Output(): name = "output" node_type = "output" - if len(node.inputs) == 1: - ref = next(iter(node.inputs.values())) + in_edges = node.in_edges + if len(in_edges) == 1: + ref = next(iter(in_edges.values())) try: value = outputs_from_loc( storage, node_location.extend_from_ref(ref), ref.port_id From a00d9c1e3020c1f168686b075a2f6207717bcb02 Mon Sep 17 00:00:00 2001 From: John Children Date: Fri, 9 Jan 2026 16:40:06 +0000 Subject: [PATCH 3/6] chore: Update stub lockfiles --- examples/example_workers/chemistry_worker/uv.lock | 12 ++++++++++-- examples/example_workers/error_worker/uv.lock | 12 ++++++++++-- examples/example_workers/hello_world_worker/uv.lock | 12 ++++++++++-- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/examples/example_workers/chemistry_worker/uv.lock b/examples/example_workers/chemistry_worker/uv.lock index 6cb38f790..d9f4802b5 100644 --- a/examples/example_workers/chemistry_worker/uv.lock +++ b/examples/example_workers/chemistry_worker/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.12" [[package]] @@ -332,10 +332,14 @@ version = "2.0.9" source = { editable = "../../../tierkreis" } dependencies = [ { name = "pydantic" }, + { name = "tierkreis-core" }, ] [package.metadata] -requires-dist = [{ name = "pydantic", specifier = "~=2.5" }] +requires-dist = [ + { name = "pydantic", specifier = "~=2.5" }, + { name = "tierkreis-core", editable = "../../../tierkreis_core" }, +] [package.metadata.requires-dev] build = [{ name = "build", extras = ["uv"] }] @@ -344,6 +348,10 @@ dev = [ { name = "pytest" }, ] +[[package]] +name = "tierkreis-core" +source = { editable = "../../../tierkreis_core" } + [[package]] name = "typing-extensions" version = "4.15.0" diff --git a/examples/example_workers/error_worker/uv.lock b/examples/example_workers/error_worker/uv.lock index cf9a2e071..458283d09 100644 --- a/examples/example_workers/error_worker/uv.lock +++ b/examples/example_workers/error_worker/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.12" [[package]] @@ -146,10 +146,14 @@ version = "2.0.9" source = { editable = "../../../tierkreis" } dependencies = [ { name = "pydantic" }, + { name = "tierkreis-core" }, ] [package.metadata] -requires-dist = [{ name = "pydantic", specifier = "~=2.5" }] +requires-dist = [ + { name = "pydantic", specifier = "~=2.5" }, + { name = "tierkreis-core", editable = "../../../tierkreis_core" }, +] [package.metadata.requires-dev] build = [{ name = "build", extras = ["uv"] }] @@ -158,6 +162,10 @@ dev = [ { name = "pytest" }, ] +[[package]] +name = "tierkreis-core" +source = { editable = "../../../tierkreis_core" } + [[package]] name = "typing-extensions" version = "4.15.0" diff --git a/examples/example_workers/hello_world_worker/uv.lock b/examples/example_workers/hello_world_worker/uv.lock index 586878db4..d5cdae586 100644 --- a/examples/example_workers/hello_world_worker/uv.lock +++ b/examples/example_workers/hello_world_worker/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.12" [[package]] @@ -146,10 +146,14 @@ version = "2.0.9" source = { editable = "../../../tierkreis" } dependencies = [ { name = "pydantic" }, + { name = "tierkreis-core" }, ] [package.metadata] -requires-dist = [{ name = "pydantic", specifier = "~=2.5" }] +requires-dist = [ + { name = "pydantic", specifier = "~=2.5" }, + { name = "tierkreis-core", editable = "../../../tierkreis_core" }, +] [package.metadata.requires-dev] build = [{ name = "build", extras = ["uv"] }] @@ -158,6 +162,10 @@ dev = [ { name = "pytest" }, ] +[[package]] +name = "tierkreis-core" +source = { editable = "../../../tierkreis_core" } + [[package]] name = "typing-extensions" version = "4.15.0" From e16316d89fcfdfc422b805a3a1845fa9fd45b60e Mon Sep 17 00:00:00 2001 From: John Children Date: Fri, 9 Jan 2026 16:40:06 +0000 Subject: [PATCH 4/6] chore: Remove unused code --- tierkreis/tierkreis/controller/start.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tierkreis/tierkreis/controller/start.py b/tierkreis/tierkreis/controller/start.py index 401f722f0..bfeb6d6c1 100644 --- a/tierkreis/tierkreis/controller/start.py +++ b/tierkreis/tierkreis/controller/start.py @@ -127,12 +127,6 @@ def start( storage.mark_node_finished(node_location) case NodeDef.Eval(): - body_loc = parent.extend_from_ref(node.body) - message = storage.read_output(body_loc, node.body.port_id) - g = ptype_from_bytes(message, GraphData) - # ins["body"] = (body_loc, node.body.port_id) - ins.update(g.fixed_inputs) - pipe_inputs_to_output_location(storage, node_location.exterior(), ins) case NodeDef.Loop(): @@ -140,7 +134,6 @@ def start( node.name is not None ): # should we do this only in debug mode? -> need to think through how this would work storage.write_debug_data(node.name, node_location) - ins["body"] = (parent.extend_from_ref(node.body), node.body.port_id) pipe_inputs_to_output_location(storage, node_location.exterior(), ins) start( storage, @@ -159,10 +152,6 @@ def start( storage.mark_node_finished(node_location) for idx, p in map_eles: eval_inputs: dict[PortID, tuple[Loc, PortID]] = {} - eval_inputs["body"] = ( - parent.extend_from_ref(node.body), - node.body.port_id, - ) for k, (i, port) in ins.items(): if port == "*": eval_inputs[k] = (i, p) From 0dd7f3690c53fa77fe516bd4c9e6a2f3de5634cd Mon Sep 17 00:00:00 2001 From: John Children Date: Fri, 9 Jan 2026 16:40:06 +0000 Subject: [PATCH 5/6] chore: Tidy up code a bit more --- devenv.nix | 1 + tierkreis/tests/cli/test_tkr.py | 1 - tierkreis/tests/controller/test_locs.py | 16 +++++ .../tierkreis/controller/data/location.py | 3 - tierkreis/tierkreis/controller/start.py | 3 +- tierkreis/tierkreis/storage.py | 8 +-- .../python/tierkreis_core/aliases.pyi | 14 ----- .../rust/bin/tierkreis-core-stubs-gen.rs | 13 ++-- tierkreis_core/rust/graph.rs | 21 ++----- .../tierkreis_visualization/data/graph.py | 6 +- .../routers/navigation.py | 19 ------ .../routers/workflows.py | 61 +++++++++---------- 12 files changed, 64 insertions(+), 102 deletions(-) delete mode 100644 tierkreis_core/python/tierkreis_core/aliases.pyi delete mode 100644 tierkreis_visualization/tierkreis_visualization/routers/navigation.py diff --git a/devenv.nix b/devenv.nix index dbab470b4..d6e77e887 100644 --- a/devenv.nix +++ b/devenv.nix @@ -5,6 +5,7 @@ packages = [ pkgs.just pkgs.graphviz + pkgs.zlib ] ++ lib.optionals pkgs.stdenv.isDarwin (with pkgs.darwin.apple_sdk; [ frameworks.CoreServices frameworks.CoreFoundation diff --git a/tierkreis/tests/cli/test_tkr.py b/tierkreis/tests/cli/test_tkr.py index 89aa36c7d..67e3a5685 100644 --- a/tierkreis/tests/cli/test_tkr.py +++ b/tierkreis/tests/cli/test_tkr.py @@ -113,7 +113,6 @@ def test_load_inputs_invalid() -> None: "args,result", cli_params, ids=["simple_eval_cli", "factorial_cli"] ) def test_end_to_end(args: list[str], result: dict[str, bytes]) -> None: - print(simple_eval().model_dump_json()) with mock.patch.object(sys, "argv", args): main() for key, value in result.items(): diff --git a/tierkreis/tests/controller/test_locs.py b/tierkreis/tests/controller/test_locs.py index 348fa1f65..8b86f3385 100644 --- a/tierkreis/tests/controller/test_locs.py +++ b/tierkreis/tests/controller/test_locs.py @@ -169,6 +169,22 @@ def test_last_step_exterior(node_location: Loc, expectation: bool) -> None: assert node_location.last_step_exterior() == expectation +@pytest.mark.parametrize( + ["node_location", "index"], + [ + (node_location_1, None), + (node_location_2, None), + (node_location_3, None), + (node_location_4, None), + (Loc().exterior(), None), + (Loc().L(1), 1), + (Loc().L(4), 4), + ], +) +def test_get_last_index(node_location: Loc, index: int | None) -> None: + assert node_location.peek_index() == index + + @pytest.mark.parametrize( ["node_location", "expected"], [ diff --git a/tierkreis/tierkreis/controller/data/location.py b/tierkreis/tierkreis/controller/data/location.py index 1cc4df7e4..4fa0b1a5d 100644 --- a/tierkreis/tierkreis/controller/data/location.py +++ b/tierkreis/tierkreis/controller/data/location.py @@ -1,12 +1,9 @@ -from logging import getLogger from pathlib import Path from typing import Optional from pydantic import BaseModel from tierkreis_core import Loc, PortID -logger = getLogger(__name__) - class WorkerCallArgs(BaseModel): function_name: str diff --git a/tierkreis/tierkreis/controller/start.py b/tierkreis/tierkreis/controller/start.py index bfeb6d6c1..ffe5f4608 100644 --- a/tierkreis/tierkreis/controller/start.py +++ b/tierkreis/tierkreis/controller/start.py @@ -5,14 +5,13 @@ import subprocess import sys -from tierkreis.controller.data.types import bytes_from_ptype, ptype_from_bytes +from tierkreis.controller.data.types import bytes_from_ptype from tierkreis.controller.executor.in_memory_executor import InMemoryExecutor from tierkreis.controller.storage.adjacency import outputs_iter from tierkreis.consts import PACKAGE_PATH from tierkreis_core import ( PortID, ExteriorRef, - GraphData, NodeDef, new_eval_root, NodeDescription, diff --git a/tierkreis/tierkreis/storage.py b/tierkreis/tierkreis/storage.py index 83f377a14..a989d64bd 100644 --- a/tierkreis/tierkreis/storage.py +++ b/tierkreis/tierkreis/storage.py @@ -20,11 +20,11 @@ def read_outputs( if isinstance(g, GraphBuilder): g = g.get_data() - output_idx = g.output_idx() - if output_idx is None: - raise ValueError("Cannot read outputs of a graph with no Output node.") + graph_outputs = g.graph_outputs() + if graph_outputs is None: + raise ValueError("Cannot read outputs of a graph with no outputs.") - out_ports = list(g.get_nodedef(output_idx).in_edges.keys()) + out_ports = list(graph_outputs.keys()) if len(out_ports) == 1 and "value" in out_ports: return ptype_from_bytes(storage.read_output(Loc(), "value")) return {k: ptype_from_bytes(storage.read_output(Loc(), k)) for k in out_ports} diff --git a/tierkreis_core/python/tierkreis_core/aliases.pyi b/tierkreis_core/python/tierkreis_core/aliases.pyi deleted file mode 100644 index 1d07a748b..000000000 --- a/tierkreis_core/python/tierkreis_core/aliases.pyi +++ /dev/null @@ -1,14 +0,0 @@ -"""Type aliases for documentation and typechecking purposes.""" - -from typing import Mapping, Sequence - -from tierkreis_core._tierkreis_core.graph import GraphData - -# Identifiers for "ports" out or into nodes in a Graph. -type PortID = str -# Values that are acceptable for use in a Const node of a Graph. -# -# Strictly a subset of types like `tierkreis.PType`. -type Value = ( - int | float | bool | str | bytes | Sequence[Value] | Mapping[str, Value] | GraphData -) diff --git a/tierkreis_core/rust/bin/tierkreis-core-stubs-gen.rs b/tierkreis_core/rust/bin/tierkreis-core-stubs-gen.rs index 4e9a4b91c..bf05c0025 100644 --- a/tierkreis_core/rust/bin/tierkreis-core-stubs-gen.rs +++ b/tierkreis_core/rust/bin/tierkreis-core-stubs-gen.rs @@ -4,7 +4,7 @@ use regex::Regex; // `crate::graph::NodeDef` has extra class attributes that cannot be generated by // `pyo3_introspection`. So we patch them in manually from a handwritten stub. -const NODEDEF_CLASS_PATCH: &'static str = " +const NODEDEF_CLASS_PATCH: &str = " class NodeDef: Func = tierkreis_core.nodes.NodeDef_Func Eval = tierkreis_core.nodes.NodeDef_Eval @@ -19,7 +19,7 @@ class NodeDef: // `crate::graph::NodeStep` has extra class attributes that cannot be generated by // `pyo3_introspection`. So we patch them in manually from a handwritten stub. -const NODESTEP_CLASS_PATCH: &'static str = " +const NODESTEP_CLASS_PATCH: &str = " class NodeStep: Root = tierkreis_core.steps.NodeStep_Root Node = tierkreis_core.steps.NodeStep_Node @@ -29,6 +29,9 @@ class NodeStep: "; pub fn main() { + let eq_regex = Regex::new(r"__eq__\(self, /, other: \w+\.\w+").unwrap(); + let ne_regex = Regex::new(r"__ne__\(self, /, other: \w+\.\w+").unwrap(); + // This assumes the package is installed as editable. // // TODO: perhaps check for venv variable instead? @@ -45,7 +48,7 @@ pub fn main() { // This is technically correct but doesn't play nicely with pyright, so patch them for now. let mut patched_stub = stub.replace("__new__(cls", "__init__(self"); - for (stub_name, _) in &stubs { + for stub_name in stubs.keys() { let module_name = stub_name.file_stem().unwrap().to_str().unwrap(); patched_stub = patched_stub.replace( &format!("import {}", module_name), @@ -103,12 +106,10 @@ pub fn main() { } // Python expects the type of the target in __eq__ to actually be object - let eq_regex = Regex::new(r"__eq__\(self, /, other: \w+\.\w+").unwrap(); let patched_stub = eq_regex.replace_all(&patched_stub, "__eq__(self, /, other: object"); // Python expects the type of the target in __ne__ to actually be object - let eq_regex = Regex::new(r"__ne__\(self, /, other: \w+\.\w+").unwrap(); - let patched_stub = eq_regex.replace_all(&patched_stub, "__ne__(self, /, other: object"); + let patched_stub = ne_regex.replace_all(&patched_stub, "__ne__(self, /, other: object"); let mut path = PathBuf::new(); path.push("python"); diff --git a/tierkreis_core/rust/graph.rs b/tierkreis_core/rust/graph.rs index b4ff4b6a5..9a30d1b2a 100644 --- a/tierkreis_core/rust/graph.rs +++ b/tierkreis_core/rust/graph.rs @@ -254,7 +254,7 @@ pub mod graph { /// than a getter. #[pyo3(signature = () -> "identifiers.NodeIndex | None")] pub fn output_idx(&self) -> Option { - self.output_idx.clone() + self.output_idx } pub fn input(&mut self, name: &str) -> ValueRef { @@ -456,12 +456,9 @@ pub mod graph { #[pyo3(signature = (node_index: "identifiers.NodeIndex") -> "dict[tierkreis_core.aliases.PortID, identifiers.NodeIndex] | None")] pub fn outputs(&self, node_index: NodeIndex) -> Option> { - self.nodes.get(node_index.0).map(|(_, outputs)| { - outputs - .iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect() - }) + self.nodes + .get(node_index.0) + .map(|(_, outputs)| outputs.iter().map(|(k, v)| (k.clone(), *v)).collect()) } #[pyo3(signature = () -> "dict[tierkreis_core.aliases.PortID, identifiers.NodeIndex] | None")] @@ -566,10 +563,7 @@ pub mod graph { body: ExteriorOrValueRef::Exterior(ExteriorRef("body".to_string())), inputs: inputs.clone(), }, - outputs: outputs - .iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(), + outputs: outputs.iter().map(|(k, v)| (k.clone(), *v)).collect(), outer_graph: Some(const_graph.clone()), }); } @@ -582,10 +576,7 @@ pub mod graph { } _ => Ok(NodeDescription { definition: nodedef.clone(), - outputs: outputs - .iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(), + outputs: outputs.iter().map(|(k, v)| (k.clone(), *v)).collect(), outer_graph: Some(self.clone()), }), } diff --git a/tierkreis_visualization/tierkreis_visualization/data/graph.py b/tierkreis_visualization/tierkreis_visualization/data/graph.py index 75fdbd725..197f4e7a1 100644 --- a/tierkreis_visualization/tierkreis_visualization/data/graph.py +++ b/tierkreis_visualization/tierkreis_visualization/data/graph.py @@ -9,13 +9,9 @@ from tierkreis_visualization.routers.models import PyGraph -def parse_node_location(node_location_str: str) -> Loc: - return Loc(node_location_str) - - def get_errored_nodes(storage: ControllerStorage) -> list[Loc]: errored_nodes = storage.read_errors(Loc("-")) - return [parse_node_location(node) for node in errored_nodes.split("\n")] + return [Loc(node) for node in errored_nodes.split("\n")] def get_node_data(storage: ControllerStorage, loc: Loc) -> PyGraph: diff --git a/tierkreis_visualization/tierkreis_visualization/routers/navigation.py b/tierkreis_visualization/tierkreis_visualization/routers/navigation.py deleted file mode 100644 index bde253b5b..000000000 --- a/tierkreis_visualization/tierkreis_visualization/routers/navigation.py +++ /dev/null @@ -1,19 +0,0 @@ -from uuid import UUID - -from tierkreis_core import Loc - - -def breadcrumb_links(crumbs: list[str]) -> list[tuple[str, str]]: - url_path = "" - links: list[tuple[str, str]] = [] - for crumb in crumbs: - url_path = url_path + crumb - links.append((crumb, url_path)) - - return links - - -def breadcrumbs(workflow_id: UUID, node_location: Loc) -> list[tuple[str, str]]: - node_location_strs: list[str] = [str(x) for x in node_location.steps()] - static_links = ["/workflows", f"/{workflow_id}/nodes/-"] - return breadcrumb_links(static_links + node_location_strs[1:]) diff --git a/tierkreis_visualization/tierkreis_visualization/routers/workflows.py b/tierkreis_visualization/tierkreis_visualization/routers/workflows.py index 607dd7b22..166c29a33 100644 --- a/tierkreis_visualization/tierkreis_visualization/routers/workflows.py +++ b/tierkreis_visualization/tierkreis_visualization/routers/workflows.py @@ -11,9 +11,9 @@ from tierkreis.controller.data.location import Loc from tierkreis.controller.storage.protocol import ControllerStorage from tierkreis_visualization.app_config import Request -from tierkreis_visualization.data.graph import get_node_data, parse_node_location +from tierkreis_visualization.data.graph import get_node_data from tierkreis_visualization.data.outputs import outputs_from_loc -from watchfiles import awatch # type: ignore +from watchfiles import awatch from tierkreis_visualization.data.workflows import WorkflowDisplay, get_workflows from tierkreis_visualization.routers.models import GraphsResponse, PyGraph @@ -22,36 +22,37 @@ logger = logging.getLogger(__name__) -@router.websocket("/{workflow_id}/nodes/{node_location_str}") +@router.websocket("/{workflow_id}/nodes/{node_location}") async def websocket_endpoint( - websocket: WebSocket, workflow_id: UUID, node_location_str: str + websocket: WebSocket, workflow_id: UUID, node_location: Loc ) -> None: if workflow_id.int == 0: return storage = websocket.app.state.get_storage_fn(workflow_id) try: await websocket.accept() - await handle_websocket(websocket, node_location_str, storage) + await handle_websocket(websocket, node_location, storage) except WebSocketDisconnect: pass async def handle_websocket( websocket: WebSocket, - node_location_str: str, + node_location: Loc, storage: ControllerStorage, ) -> None: - node_location = parse_node_location(node_location_str) - async for changes in awatch(storage.workflow_dir, recursive=True): relevant_changes: set[str] = set() for change in changes: path = Path(change[1]).relative_to(storage.workflow_dir) if not path.parts: continue - loc = path.parts[0] - if loc.startswith(str(node_location)): - relevant_changes.add(loc) + # This will include files that are not completely described + # by a Loc, so we check for string prefixes rather than + # using the Loc data structure directly. + path_prefix = path.parts[0] + if path_prefix.startswith(str(node_location)): + relevant_changes.add(str(path_prefix)) if relevant_changes: await websocket.send_json(list(relevant_changes)) @@ -77,35 +78,32 @@ def list_nodes( return GraphsResponse(graphs={loc: get_node_data(storage, loc) for loc in locs}) -@router.get("/{workflow_id}/nodes/{node_location_str}") -def get_node(request: Request, workflow_id: UUID, node_location_str: str) -> PyGraph: - node_location = parse_node_location(node_location_str) +@router.get("/{workflow_id}/nodes/{node_location}") +def get_node(request: Request, workflow_id: UUID, node_location: Loc) -> PyGraph: storage = request.app.state.get_storage_fn(workflow_id) return get_node_data(storage, node_location) -@router.get("/{workflow_id}/nodes/{node_location_str}/outputs") +@router.get("/{workflow_id}/nodes/{node_location}/outputs") def get_eval_outputs( request: Request, workflow_id: UUID, - node_location_str: str, + node_location: Loc, ): - loc = parse_node_location(node_location_str) storage = request.app.state.get_storage_fn(workflow_id) - outputs = storage.read_output_ports(loc) - out = {k: str(storage.read_output(loc, k)) for k in outputs} + outputs = storage.read_output_ports(node_location) + out = {k: str(storage.read_output(node_location, k)) for k in outputs} return JSONResponse(out) -@router.get("/{workflow_id}/nodes/{node_location_str}/inputs/{port_name}") +@router.get("/{workflow_id}/nodes/{node_location}/inputs/{port_name}") def get_input( request: Request, workflow_id: UUID, - node_location_str: str, + node_location: Loc, port_name: str, ): try: - node_location = parse_node_location(node_location_str) storage = request.app.state.get_storage_fn(workflow_id) definition = storage.read_worker_call_args(node_location) @@ -115,16 +113,15 @@ def get_input( return PlainTextResponse(str(e)) -@router.get("/{workflow_id}/nodes/{node_location_str}/outputs/{port_name}") +@router.get("/{workflow_id}/nodes/{node_location}/outputs/{port_name}") def get_output( request: Request, workflow_id: UUID, - node_location_str: str, + node_location: Loc, port_name: str, ): - loc = parse_node_location(node_location_str) storage = request.app.state.get_storage_fn(workflow_id) - return PlainTextResponse(outputs_from_loc(storage, loc, port_name)) + return PlainTextResponse(outputs_from_loc(storage, node_location, port_name)) @router.get("/{workflow_id}/logs") @@ -137,13 +134,12 @@ def get_logs( return PlainTextResponse(logs) -@router.get("/{workflow_id}/nodes/{node_location_str}/errors") +@router.get("/{workflow_id}/nodes/{node_location}/errors") def get_errors( request: Request, workflow_id: UUID, - node_location_str: str, + node_location: Loc, ): - node_location = parse_node_location(node_location_str) storage = request.app.state.get_storage_fn(workflow_id) if not storage.node_has_error(node_location): return PlainTextResponse("Node has no errors.", status_code=404) @@ -152,8 +148,7 @@ def get_errors( return PlainTextResponse(messages) -@router.post("/{workflow_id}/nodes/{node_location_str}/restart") -def restart(request: Request, workflow_id: UUID, node_location_str: str) -> list[Loc]: - loc = parse_node_location(node_location_str) +@router.post("/{workflow_id}/nodes/{node_location}/restart") +def restart(request: Request, workflow_id: UUID, node_location: Loc) -> list[Loc]: storage: ControllerStorage = request.app.state.get_storage_fn(workflow_id) - return storage.restart_task(loc) + return storage.restart_task(node_location) From 557312340b468ee46532f475093c60b7e51da3dc Mon Sep 17 00:00:00 2001 From: John Children Date: Mon, 12 Jan 2026 15:13:28 +0000 Subject: [PATCH 6/6] chore: Update comments --- tierkreis/tierkreis/controller/storage/adjacency.py | 8 +++++++- tierkreis/tierkreis/controller/storage/walk.py | 3 ++- tierkreis_core/python/tierkreis_core/aliases.py | 4 +++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/tierkreis/tierkreis/controller/storage/adjacency.py b/tierkreis/tierkreis/controller/storage/adjacency.py index 21f7014f0..cce3e4934 100644 --- a/tierkreis/tierkreis/controller/storage/adjacency.py +++ b/tierkreis/tierkreis/controller/storage/adjacency.py @@ -10,8 +10,14 @@ def unfinished_inputs( storage: ControllerStorage, loc: Loc, node: NodeDef ) -> list[ValueRef]: + """Retrieve the ValueRefs for each edge of the graph that isn't finished. + + As ExteriorRefs always refer to edges that are finished they can be safely ingored. + """ ins = node.in_edges.values() - ins = [x for x in ins if isinstance(x, ValueRef)] # Only look an Values on Edges + ins = [ + x for x in ins if isinstance(x, ValueRef) + ] # ExteriorRefs are always finished. return [x for x in ins if not storage.is_node_finished(loc.N(x.node_index))] diff --git a/tierkreis/tierkreis/controller/storage/walk.py b/tierkreis/tierkreis/controller/storage/walk.py index dff9585cb..8cdd5d05a 100644 --- a/tierkreis/tierkreis/controller/storage/walk.py +++ b/tierkreis/tierkreis/controller/storage/walk.py @@ -164,7 +164,8 @@ def walk_loop( previous_index = new_location.peek_index() if previous_index is None: - # TODO: This should be impossible + # This should be impossible as we expect `latest_loop_iteration` + # to always return a loop step. raise ValueError("Previous step is not a Loop step.") # The outputs from the previous iteration diff --git a/tierkreis_core/python/tierkreis_core/aliases.py b/tierkreis_core/python/tierkreis_core/aliases.py index 31c152472..0e7f6794d 100644 --- a/tierkreis_core/python/tierkreis_core/aliases.py +++ b/tierkreis_core/python/tierkreis_core/aliases.py @@ -9,7 +9,9 @@ from tierkreis_core import GraphData type PortID = str -# Constant values +# This `Value` type refers to data that can be used in a constant node. +# +# Note that this is a subset of `PType` from `tierkreis`. type Value = ( int | float | bool | str | bytes | Sequence[Value] | Mapping[str, Value] | GraphData )