diff --git a/docs/source/_static/first_graph.png b/docs/source/_static/first_graph.png new file mode 100644 index 000000000..754314d18 Binary files /dev/null and b/docs/source/_static/first_graph.png differ diff --git a/docs/source/_static/first_graph_overview.png b/docs/source/_static/first_graph_overview.png new file mode 100644 index 000000000..4b8dfd197 Binary files /dev/null and b/docs/source/_static/first_graph_overview.png differ diff --git a/docs/source/conf.py b/docs/source/conf.py index 3867485fb..907cfd36e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -12,32 +12,33 @@ # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration - -extensions = ["autodoc2", "myst_nb", "sphinx.ext.intersphinx"] +add_module_names = False +extensions = ["autodoc2", "myst_nb", "sphinx.ext.intersphinx", "sphinx.ext.mathjax"] +latex_engine = "xelatex" autodoc2_packages = [ "../../tierkreis/tierkreis", { - "path": "../../tierkreis_workers/aer_worker/src/impl/aer_worker_impl.py", + "path": "../../tierkreis_workers/aer_worker/tkr_aer_worker_impl/impl.py", "module": "aer_worker", }, { - "path": "../../tierkreis_workers/ibmq_worker/src/impl/ibmq_worker_impl.py", - "module": "ibmq_worker", + "path": "../../tierkreis_workers/ibmq_worker/tkr_ibmq_worker_impl/impl.py", + "module": "ibmq_worker.tkr_ibmq_worker_impl.impl", }, { - "path": "../../tierkreis_workers/nexus_worker/src/impl/nexus_worker_impl.py", + "path": "../../tierkreis_workers/nexus_worker/tkr_nexus_worker_impl/impl.py", "module": "nexus_worker", }, { - "path": "../../tierkreis_workers/pytket_worker/src/impl/pytket_worker_impl.py", - "module": "pytket_worker", + "path": "../../tierkreis_workers/pytket_worker/tkr_pytket_worker_impl/impl.py", + "module": "pytket_worker.tkr_pytket_worker_impl.impl", }, { - "path": "../../tierkreis_workers/quantinuum_worker/src/impl/quantinuum_worker_impl.py", - "module": "quantinuum_worker", + "path": "../../tierkreis_workers/quantinuum_worker/tkr_quantinuum_worker_impl/impl.py", + "module": "quantinuum_worker.tkr_quantinuum_worker_impl.impl", }, { - "path": "../../tierkreis_workers/qulacs_worker/src/impl/qulacs_worker_impl.py", + "path": "../../tierkreis_workers/qulacs_worker/tkr_qulacs_worker_impl/impl.py", "module": "qulacs_worker", }, ] @@ -54,6 +55,7 @@ "storage_and_executors.ipynb", "hpc.ipynb", ] +myst_enable_extensions = ["dollarmath", "amsmath"] nitpicky = True exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "examples/**/.venv/**"] diff --git a/docs/source/examples/api/stubs.py b/docs/source/examples/api/stubs.py new file mode 100644 index 000000000..da89e75a5 --- /dev/null +++ b/docs/source/examples/api/stubs.py @@ -0,0 +1,17 @@ +"""Code generated from hello_world_worker namespace. Please do not edit.""" + +from typing import NamedTuple +from tierkreis.controller.data.models import TKR + + +class greet(NamedTuple): + greeting: TKR[str] # noqa: F821 # fmt: skip + subject: TKR[str] # noqa: F821 # fmt: skip + + @staticmethod + def out() -> type[TKR[str]]: # noqa: F821 # fmt: skip + return TKR[str] # noqa: F821 # fmt: skip + + @property + def namespace(self) -> str: + return "hello_world_worker" diff --git a/docs/source/examples/errors_and_debugging.ipynb b/docs/source/examples/errors_and_debugging.ipynb index 3d6dd0341..c2743f7ed 100644 --- a/docs/source/examples/errors_and_debugging.ipynb +++ b/docs/source/examples/errors_and_debugging.ipynb @@ -93,7 +93,7 @@ "\n", "The first avenue for debugging is enabling fine grained logging.\n", "The tierkreis logging inherits properties from the root logger so it suffices to set a `basicConfig` which changes **only** the logger of the controller.\n", - "When running a python worker, Tierkreis will check the environment variables `$TKR_LOG_LEVEL`, `$TKR_LOG_FORMAT` and `$TKR_DATE_FORMAT` for logger information as detailed [here](../logging_and_errors.md)." + "When running a python worker, Tierkreis will check the environment variables `$TKR_LOG_LEVEL`, `$TKR_LOG_FORMAT` and `$TKR_DATE_FORMAT` for logger information as detailed [here](../tutorial/logging_and_errors.md)." ] }, { diff --git a/docs/source/examples/example_workers/multiple_outputs_worker/README.md b/docs/source/examples/example_workers/multiple_outputs_worker/README.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/docs/source/examples/example_workers/multiple_outputs_worker/api/README.md b/docs/source/examples/example_workers/multiple_outputs_worker/api/README.md deleted file mode 100644 index fa2006604..000000000 --- a/docs/source/examples/example_workers/multiple_outputs_worker/api/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# multiple_outputs Worker API - -This is the api for a custom worker you just created. -The directory contains the stubs which are used during construction of a graph. - -**You don't need to touch this code, it will be automatically generated.** - -They can be generated using the `trk init stubs` command and are not required to be present during runtime. -They are encapsulated in a separate package `tkr-multiple_outputs` to avoid unnecessary dependencies during construction of the worker. -The worker project depends on the api but not the other way around, -so you can for example use the api in your graph code without having to install the whole worker. - diff --git a/docs/source/examples/example_workers/multiple_outputs_worker/api/pyproject.toml b/docs/source/examples/example_workers/multiple_outputs_worker/api/pyproject.toml index a16b5a7c2..42df24e0f 100644 --- a/docs/source/examples/example_workers/multiple_outputs_worker/api/pyproject.toml +++ b/docs/source/examples/example_workers/multiple_outputs_worker/api/pyproject.toml @@ -1,7 +1,6 @@ [project] name = "tkr-multiple-outputs-worker" version = "0.1.0" -readme = "README.md" requires-python = ">=3.12" dependencies = [ "tierkreis", diff --git a/docs/source/examples/example_workers/multiple_outputs_worker/pyproject.toml b/docs/source/examples/example_workers/multiple_outputs_worker/pyproject.toml index fecd8ee8a..407785f9b 100644 --- a/docs/source/examples/example_workers/multiple_outputs_worker/pyproject.toml +++ b/docs/source/examples/example_workers/multiple_outputs_worker/pyproject.toml @@ -2,7 +2,6 @@ name = "tkr-multiple-outputs-worker-impl" version = "0.2.0" description = "A tierkreis worker implementation." -readme = "README.md" requires-python = ">=3.12" dependencies = [ "tierkreis", diff --git a/docs/source/examples/example_workers/pytket_example_worker/api/api.py b/docs/source/examples/example_workers/pytket_example_worker/api/api.py new file mode 100644 index 000000000..bc81d8734 --- /dev/null +++ b/docs/source/examples/example_workers/pytket_example_worker/api/api.py @@ -0,0 +1,43 @@ +"""Code generated from pytket_example_worker namespace. Please do not edit.""" + +from typing import NamedTuple +from tierkreis.controller.data.models import TKR, OpaqueType + + +class substitute(NamedTuple): + circuit: TKR[OpaqueType["pytket._tket.circuit.Circuit"]] # noqa: F821 # fmt: skip + a: TKR[float] # noqa: F821 # fmt: skip + b: TKR[float] # noqa: F821 # fmt: skip + c: TKR[float] # noqa: F821 # fmt: skip + + @staticmethod + def out() -> type[TKR[OpaqueType["pytket._tket.circuit.Circuit"]]]: # noqa: F821 # fmt: skip + return TKR[OpaqueType["pytket._tket.circuit.Circuit"]] # noqa: F821 # fmt: skip + + @property + def namespace(self) -> str: + return "pytket_example_worker" + + +class simulate(NamedTuple): + circuit: TKR[OpaqueType["pytket._tket.circuit.Circuit"]] # noqa: F821 # fmt: skip + + @staticmethod + def out() -> type[TKR[OpaqueType["pytket.backends.backendresult.BackendResult"]]]: # noqa: F821 # fmt: skip + return TKR[OpaqueType["pytket.backends.backendresult.BackendResult"]] # noqa: F821 # fmt: skip + + @property + def namespace(self) -> str: + return "pytket_example_worker" + + +class optimise(NamedTuple): + circuit: TKR[OpaqueType["pytket._tket.circuit.Circuit"]] # noqa: F821 # fmt: skip + + @staticmethod + def out() -> type[TKR[OpaqueType["pytket._tket.circuit.Circuit"]]]: # noqa: F821 # fmt: skip + return TKR[OpaqueType["pytket._tket.circuit.Circuit"]] # noqa: F821 # fmt: skip + + @property + def namespace(self) -> str: + return "pytket_example_worker" diff --git a/docs/source/examples/example_workers/pytket_example_worker/api/pyproject.toml b/docs/source/examples/example_workers/pytket_example_worker/api/pyproject.toml new file mode 100644 index 000000000..ebdfab491 --- /dev/null +++ b/docs/source/examples/example_workers/pytket_example_worker/api/pyproject.toml @@ -0,0 +1,17 @@ +[project] +name = "tkr-pytket-example-worker" +version = "0.2.0" +requires-python = ">=3.12" +dependencies = [ + "tierkreis", +] + +[tool.uv.sources] +tierkreis = { path = "../../../../../../tierkreis", editable = true } + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel.force-include] +"api.py" = "pytket_example_worker.py" diff --git a/docs/source/examples/example_workers/pytket_example_worker/pyproject.toml b/docs/source/examples/example_workers/pytket_example_worker/pyproject.toml new file mode 100644 index 000000000..efeb61a8d --- /dev/null +++ b/docs/source/examples/example_workers/pytket_example_worker/pyproject.toml @@ -0,0 +1,26 @@ +[project] +name = "tkr-pytket-example-worker-impl" +version = "0.1.0" +requires-python = ">=3.12" +dependencies = [ + "pydantic>=2.11.7", + "pytket>=2.9.3", + "tierkreis", + "ruff", + "sympy", + "pytket-qiskit", + "tkr-pytket-example-worker", +] + +[tool.uv.sources] +tierkreis = { workspace = true } +tkr-pytket-example-worker = { workspace = true } + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + + + +[project.scripts] +tkr-pytket-example-worker = "tkr_pytket_example_worker_impl.main:main" diff --git a/docs/source/examples/example_workers/pytket_example_worker/tkr_pytket_example_worker_impl/__init__.py b/docs/source/examples/example_workers/pytket_example_worker/tkr_pytket_example_worker_impl/__init__.py new file mode 100644 index 000000000..4061eba8c --- /dev/null +++ b/docs/source/examples/example_workers/pytket_example_worker/tkr_pytket_example_worker_impl/__init__.py @@ -0,0 +1,3 @@ +from .impl import worker + +__all__ = ["worker"] diff --git a/docs/source/examples/example_workers/pytket_example_worker/tkr_pytket_example_worker_impl/impl.py b/docs/source/examples/example_workers/pytket_example_worker/tkr_pytket_example_worker_impl/impl.py new file mode 100644 index 000000000..c30952556 --- /dev/null +++ b/docs/source/examples/example_workers/pytket_example_worker/tkr_pytket_example_worker_impl/impl.py @@ -0,0 +1,31 @@ +import logging + +from pytket._tket.circuit import Circuit +from pytket.backends.backendresult import BackendResult +from pytket.extensions.qiskit.backends.aer import AerBackend +from pytket.transform import Transform +from sympy import Symbol + +from tierkreis import Worker + +logger = logging.getLogger(__name__) + +worker = Worker("pytket_example_worker") + + +@worker.task() +def substitute(circuit: Circuit, a: float, b: float, c: float) -> Circuit: + circuit.symbol_substitution({Symbol("a"): a, Symbol("b"): b, Symbol("c"): c}) + return circuit + + +@worker.task() +def simulate(circuit: Circuit) -> BackendResult: + backend = AerBackend() + return backend.run_circuit(circuit, n_shots=1000) + + +@worker.task() +def optimise(circuit: Circuit) -> Circuit: + Transform.OptimisePhaseGadgets().apply(circuit) + return circuit diff --git a/docs/source/examples/example_workers/pytket_example_worker/tkr_pytket_example_worker_impl/main.py b/docs/source/examples/example_workers/pytket_example_worker/tkr_pytket_example_worker_impl/main.py new file mode 100644 index 000000000..08f1359c1 --- /dev/null +++ b/docs/source/examples/example_workers/pytket_example_worker/tkr_pytket_example_worker_impl/main.py @@ -0,0 +1,11 @@ +from sys import argv + +from tkr_pytket_example_worker_impl import worker + + +def main(): + worker.app(argv) + + +if __name__ == "__main__": + main() diff --git a/docs/source/examples/first_graph.ipynb b/docs/source/examples/first_graph.ipynb new file mode 100644 index 000000000..67b38ccf2 --- /dev/null +++ b/docs/source/examples/first_graph.ipynb @@ -0,0 +1,430 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "52132a14", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "source": [ + "# Lesson 1: A first workflow\n", + "\n", + "```{note}\n", + "Have you already installed everything you need? If you're \n", + "not sure, check the \"Before you begin\" section of the \n", + "[introduction](../index.md). \n", + "```\n", + "\n", + "In this lesson. we'll construct a very simple workflow that simply\n", + "reads three numbers from its inputs and outputs their sum. It's\n", + "nealry trivial, but this will demonstrate the essentials of working\n", + "with Tierkreis :\n", + "1. Defining a graph through its inputs and outputs\n", + "2. Constructing the computation\n", + " - Using the inputs\n", + " - Using simple nodes\n", + " - Using built-in functionality\n", + "3. Checking what we've done with the visualizer\n", + "4. Running the workflow\n", + "\n", + "Let's get started!" + ] + }, + { + "cell_type": "markdown", + "id": "86dd44d1-0a31-48c8-935b-9de1dd482f41", + "metadata": {}, + "source": [ + "## Setting up the first graph\n", + "\n", + "Workflows are defined by their *graphs* which described how the inputs produce the outputs. The first step in defining the graph is to specify those inputs and outputs. And part of that is knowing their names and their *types*.\n", + "\n", + "Types are optional in Tierkreis, but since compile-time errors are cheap, and run-time errors are expensive, we strongly recommend using them everywhere.\n", + "\n", + "```{note}\n", + "In order to keep a clear separation between the types used in the Tierkreis graph and the types already present in the Python\n", + "language we wrap the former with `TKR`. in other words, the `TKR[A]` wrapper type indicates that an edge in the graph contains a value\n", + "of type `A`. More on this in the [core concepts](../tutorial/core_concepts.md#types)\n", + "```\n", + "\n", + "In general a graph can have any number of inputs or outputs. In this example, we have three inputs, so our first step is to create a python object to hold the inputs. We always use a `NamedTuple` for this. There's nothing special about the name `InParams`, it's just intended to be descriptive.\n", + "\n", + "Since we have only one output, and we already know that the type is`TKR[float]`, we don't need to do anything for that." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5783b41e-ea9b-41c4-8ce7-406971f40908", + "metadata": {}, + "outputs": [], + "source": [ + "from typing import NamedTuple\n", + "from tierkreis.models import TKR\n", + "\n", + "\n", + "class InParams(NamedTuple):\n", + " a: TKR[float]\n", + " b: TKR[float]\n", + " c: TKR[float]" + ] + }, + { + "cell_type": "markdown", + "id": "6ac7c4f3-107f-4675-a96d-f9a4b39f50dd", + "metadata": {}, + "source": [ + "Graphs are built using a `GraphBuilder`. It needs to be instantiated with the types of its inputs and outputs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c15164f5", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "from tierkreis.builder import GraphBuilder\n", + "\n", + "g = GraphBuilder(InParams, TKR[float])" + ] + }, + { + "cell_type": "markdown", + "id": "226d8c71", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "source": [ + "## Building the Graph\n", + "\n", + "Initially the graph doesn't do anything. To change this we need to add some nodes to the graph. There are several kinds of nodes we can have - for example Inputs and Outputs are nodes - but the most useful kind are called *Tasks*. Roughly speaking a task is any kind of computation. Most tasks are done by *Workers*, but for basic things, like adding numbers together, we can rely on *built-ins*. Tierkreis has many built-ins [API docs](#tierkreis.builtins.main) but the only one we will need today is floating point addition, aka `add`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ad221389-f2d8-46cf-8644-27d1fdad6938", + "metadata": {}, + "outputs": [], + "source": [ + "from tierkreis.builtins import add" + ] + }, + { + "cell_type": "markdown", + "id": "1546d4c8-d6bd-4533-9b9f-9f44ab8ae232", + "metadata": {}, + "source": [ + "Next we're going to construct a graph by calling the builder functions.\n", + "Each function adds a node to the graph; the arguments are the input edges to the node, and the return values are its output edges. Notice that `inputs` is a special node created when we instantiated `g` and the names of the inputs are the ones we chose when defing the `InParams` class earlier." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1f891725", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "x = g.task(add(g.inputs.a, g.inputs.b))\n", + "y = g.task(add(x, g.inputs.c))" + ] + }, + { + "cell_type": "markdown", + "id": "a1279db9", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "source": [ + "Finally, we have to say which edges of our graph will be the outputs. In our example there's only one." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bfdf37b9", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "g.outputs(y)" + ] + }, + { + "cell_type": "markdown", + "id": "12670b4a-af60-404f-9e3f-cb2145c2472c", + "metadata": {}, + "source": [ + "Congratulations - you made your first graph! But what does it look like?" + ] + }, + { + "cell_type": "markdown", + "id": "90e724f1", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "source": [ + "## Using the visualizer\n", + "\n", + "Tierkreis comes with an additional library to keep track of your workflows. The main use is to observe a running workflow, but you can also use it to examine graphs that you are currently constructing.\n", + "\n", + "The visualizer will run a local web application in the same process. To stop its execution you need to user `ctrl+c`. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3c19537", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "%%script false --no-raise-error\n", + "from tierkreis_visualization.visualize_graph import visualize_graph\n", + "\n", + "visualize_graph(g)" + ] + }, + { + "cell_type": "markdown", + "id": "740edc44-bb21-47c8-8624-e8cb1346bf50", + "metadata": {}, + "source": [ + "Opening the web interface at `localhost:8000` will show the landing page with the workflow overview.\n", + "![Landing Page](../_static/first_graph_overview.png)\n", + "\n", + "After selecting the `tmp` workflow you will see the graph representation you just created.\n", + "![Graph](../_static/first_graph.png)\n", + "\n", + "It shows the three input nodes `a,b,c`, the two task nodes `builtins.add` and an output with value `null` as the workflow hasn't run.\n", + "For the same reason all the nodes are depicted in white, which means they haven't been started yet.\n", + "\n", + "To learn more about the visualizer see [this page](../tutorial/visualization.md)" + ] + }, + { + "cell_type": "markdown", + "id": "9f86ba49", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "source": [ + "## Running the graph\n", + "\n", + "Now we have made the graph, and checked that it looks like we expect, it's time to run it.\n", + "\n", + "Tierkreis can run in a lot of complex configurations, but for this tutorial we will be running it locally. It's pretty simple, but there will be some code that we won't explain here. To run a general Tierkreis graph we need to set up:\n", + "\n", + "- a way to store and share input and output values (the 'storage' interface)\n", + "- a way to run tasks (the 'executor' interface)\n", + "\n", + "For this example we use the `FileStorage` that is provided by the Tierkreis library itself.\n", + "The inputs and outputs will be stored in a directory on disk.\n", + "(By default the files are stored in `~/.tierkreis/checkpoints/`, where `` is a `UUID` identifying the workflow.)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "48846186", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "from uuid import UUID\n", + "\n", + "\n", + "from tierkreis.storage import FileStorage\n", + "\n", + "storage = FileStorage(workflow_id=UUID(int=12345), name=\"Hello World Graph\")" + ] + }, + { + "cell_type": "markdown", + "id": "ac3c1225", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "source": [ + "If we have already run this example then there will already be files at this directory in the storage.\n", + "If we want to reuse the directory then run" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4757171", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "storage.clean_graph_files()" + ] + }, + { + "cell_type": "markdown", + "id": "ec90fb03", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "source": [ + "to get a fresh area to work in.\n", + "\n", + "Since we are just using the Tierkreis built-in tasks the executor will not actually be called.\n", + "As a placeholder we create a simple `ShellExecutor`, also provided by the Tierkreis library, which can run bash scripts in a specified directory. In this case we can use `None`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "925fbce0", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "from tierkreis.executor import ShellExecutor\n", + "\n", + "executor = ShellExecutor(registry_path=None, workflow_dir=storage.workflow_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "bb1a9769", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "source": [ + "As the penultimate step we need to provide the workflow inputs to run as a dictionary which we get from the input class..\n", + "If the inputs are not provided the workflow will encounter an error." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ecc65977", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "inputs = InParams(0, 0.25, 0.5)._asdict()" + ] + }, + { + "cell_type": "markdown", + "id": "66845ea8", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "source": [ + "\n", + "With the storage and executor specified and inputs set, we can now run a graph using `run_graph`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ca96cb3d", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "from tierkreis.controller import run_graph\n", + "from tierkreis.storage import read_outputs\n", + "\n", + "run_graph(storage, executor, g, inputs)\n", + "result = read_outputs(g, storage)\n", + "print(result)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "tierkreis", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.11" + }, + "name": "first_graph.ipynb" + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/source/examples/hamiltonian.ipynb b/docs/source/examples/hamiltonian.ipynb index 177d13f5a..9efd05b92 100644 --- a/docs/source/examples/hamiltonian.ipynb +++ b/docs/source/examples/hamiltonian.ipynb @@ -1,376 +1,408 @@ { - "cells": [ - { - "cell_type": "markdown", - "id": "d56e6c6d", - "metadata": {}, - "source": [ - "# Hamiltonian Simulation\n", - "\n", - "In this example we're going to apply the previously learned concept to run a hamiltonian simulation.\n", - "As before we are going to define a symbolic circuit as an ansatz." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3563e447", - "metadata": {}, - "outputs": [], - "source": [ - "%pip install tierkreis pytket" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bfbaa4df", - "metadata": {}, - "outputs": [], - "source": [ - "from pytket._tket.circuit import Circuit, fresh_symbol\n", - "\n", - "\n", - "def build_ansatz() -> Circuit:\n", - " a = fresh_symbol(\"a\")\n", - " b = fresh_symbol(\"b\")\n", - " c = fresh_symbol(\"c\")\n", - " circ = Circuit(4)\n", - " circ.CX(0, 1)\n", - " circ.CX(1, 2)\n", - " circ.CX(2, 3)\n", - " circ.Rz(a, 3)\n", - " circ.CX(2, 3)\n", - " circ.CX(1, 2)\n", - " circ.CX(0, 1)\n", - " circ.Rz(b, 0)\n", - " circ.CX(0, 1)\n", - " circ.CX(1, 2)\n", - " circ.CX(2, 3)\n", - " circ.Rz(c, 3)\n", - " circ.CX(2, 3)\n", - " circ.CX(1, 2)\n", - " circ.CX(0, 1)\n", - " return circ" - ] - }, - { - "cell_type": "markdown", - "id": "d1993864", - "metadata": {}, - "source": [ - "We are going to simulate a hamiltonian given as a list of Pauli strings and their weights.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f5d2e7be", - "metadata": {}, - "outputs": [], - "source": [ - "from tierkreis.builder import GraphBuilder\n", - "from tierkreis.controller.data.models import TKR\n", - "from typing import NamedTuple, Literal\n", - "\n", - "from substitution_worker import substitute\n", - "\n", - "\n", - "class SymbolicExecutionInputs(NamedTuple):\n", - " a: TKR[float]\n", - " b: TKR[float]\n", - " c: TKR[float]\n", - " ham: TKR[list[tuple[Literal[\"pytket._tket.pauli.QubitPauliString\"], float]]]\n", - " ansatz: TKR[Literal[\"pytket._tket.circuit.Circuit\"]]\n", - "\n", - "\n", - "simulation_graph = GraphBuilder(SymbolicExecutionInputs, TKR[float])\n", - "substituted_circuit = simulation_graph.task(\n", - " substitute(\n", - " simulation_graph.inputs.ansatz,\n", - " simulation_graph.inputs.a,\n", - " simulation_graph.inputs.b,\n", - " simulation_graph.inputs.c,\n", - " )\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "161a5e02", - "metadata": {}, - "source": [ - "We will evaluate this circuit with an observable based on a Pauli string" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "39b21062", - "metadata": {}, - "outputs": [], - "source": [ - "from tierkreis.pytket_worker import (\n", - " append_pauli_measurement_impl,\n", - " optimise_phase_gadgets,\n", - " expectation,\n", - ")\n", - "from tierkreis.aer_worker import submit_single\n", - "\n", - "\n", - "class SubmitInputs(NamedTuple):\n", - " circuit: TKR[Literal[\"pytket._tket.circuit.Circuit\"]]\n", - " pauli_string: TKR[Literal[\"pytket._tket.pauli.QubitPauliString\"]]\n", - " n_shots: TKR[int]\n", - "\n", - "\n", - "def exp_val():\n", - " g = GraphBuilder(SubmitInputs, TKR[float])\n", - "\n", - " circuit = g.inputs.circuit\n", - " pauli_string = g.inputs.pauli_string\n", - " n_shots = g.inputs.n_shots\n", - "\n", - " measurement_circuit = g.task(append_pauli_measurement_impl(circuit, pauli_string))\n", - "\n", - " compiled_circuit = g.task(optimise_phase_gadgets(measurement_circuit))\n", - "\n", - " backend_result = g.task(submit_single(compiled_circuit, n_shots))\n", - " av = g.task(expectation(backend_result))\n", - " g.outputs(av)\n", - " return g" - ] - }, - { - "cell_type": "markdown", - "id": "c7f5f9a1", - "metadata": {}, - "source": [ - "Since `exp_val` runs independently for each Pauli string we can implement this using a map, \n", - "but we need to prepare the inputs first." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "021e0e5b", - "metadata": {}, - "outputs": [], - "source": [ - "from tierkreis.builtins import unzip\n", - "from tierkreis.controller.data.models import TKR\n", - "\n", - "pauli_strings_list, parameters_list = simulation_graph.task(\n", - " unzip(simulation_graph.inputs.ham)\n", - ")\n", - "input_circuits = simulation_graph.map(\n", - " lambda x: SubmitInputs(substituted_circuit, x, simulation_graph.const(100)),\n", - " pauli_strings_list,\n", - ")\n", - "exp_values = simulation_graph.map(exp_val(), input_circuits)" - ] - }, - { - "cell_type": "markdown", - "id": "7788777e", - "metadata": {}, - "source": [ - "To estimate the energy, we can take a weighted sum of the expectation values.\n", - "For this we want to compute a reduction (\\(x,y) \\z --> x*y+z) which we implement as a fold function." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cd24e08c", - "metadata": {}, - "outputs": [], - "source": [ - "from tierkreis.builtins import add, times, untuple\n", - "from tierkreis.graphs.fold import FoldFunctionInput\n", - "\n", - "ComputeTermsInputs = FoldFunctionInput[\n", - " tuple[float, float], float\n", - "] # (value, accum) -> new_accum\n", - "\n", - "\n", - "def compute_terms():\n", - " g = GraphBuilder(ComputeTermsInputs, TKR[float])\n", - "\n", - " res_0, res_1 = g.task(untuple(g.inputs.value))\n", - " prod = g.task(times(res_0, res_1))\n", - " sum = g.task(add(g.inputs.accum, prod))\n", - "\n", - " g.outputs(sum)\n", - " return g" - ] - }, - { - "cell_type": "markdown", - "id": "af57c8ac", - "metadata": {}, - "source": [ - "Preparing the inputs for the fold graph we are going to use tuples (x=exp_val, y=weight) and defining the start value z=0." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9bab0de3", - "metadata": {}, - "outputs": [], - "source": [ - "from tierkreis.builtins import tkr_zip\n", - "from tierkreis.graphs.fold import FoldGraphInputs, fold_graph\n", - "\n", - "tuple_values = simulation_graph.task(tkr_zip(exp_values, parameters_list))\n", - "fold_inputs = FoldGraphInputs(simulation_graph.const(0.0), tuple_values)" - ] - }, - { - "cell_type": "markdown", - "id": "da997ea2", - "metadata": {}, - "source": [ - "applying the fold operation yields the final output" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d7c6fa18", - "metadata": {}, - "outputs": [], - "source": [ - "computed = simulation_graph.eval(fold_graph(compute_terms()), fold_inputs)\n", - "simulation_graph.outputs(computed)" - ] - }, - { - "cell_type": "markdown", - "id": "5999c0a5", - "metadata": {}, - "source": [ - "As before we now have to set up tierkreis storage and executors" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "aec32e19", - "metadata": {}, - "outputs": [], - "source": [ - "from pathlib import Path\n", - "from uuid import UUID\n", - "\n", - "from tierkreis.consts import PACKAGE_PATH\n", - "from tierkreis.controller.executor.multiple import MultipleExecutor\n", - "from tierkreis.storage import FileStorage\n", - "from tierkreis.controller.executor.uv_executor import UvExecutor\n", - "\n", - "storage = FileStorage(UUID(int=102), name=\"hamiltonian\")\n", - "example_executor = UvExecutor(\n", - " registry_path=Path().parent / \"example_workers\", logs_path=storage.logs_path\n", - ")\n", - "common_executor = UvExecutor(\n", - " registry_path=PACKAGE_PATH.parent / \"tierkreis_workers\", logs_path=storage.logs_path\n", - ")\n", - "multi_executor = MultipleExecutor(\n", - " common_executor,\n", - " executors={\"custom\": example_executor},\n", - " assignments={\"substitution_worker\": \"custom\"},\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "20848215", - "metadata": {}, - "source": [ - "and provide the inputs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2d3b86c0", - "metadata": {}, - "outputs": [], - "source": [ - "from pytket._tket.unit_id import Qubit\n", - "from pytket.pauli import Pauli, QubitPauliString\n", - "\n", - "qubits = [Qubit(0), Qubit(1), Qubit(2), Qubit(3)]\n", - "hamiltonian = [\n", - " (QubitPauliString(qubits, [Pauli.X, Pauli.Y, Pauli.X, Pauli.I]).to_list(), 0.1),\n", - " (QubitPauliString(qubits, [Pauli.Y, Pauli.Z, Pauli.X, Pauli.Z]).to_list(), 0.5),\n", - " (QubitPauliString(qubits, [Pauli.X, Pauli.Y, Pauli.Z, Pauli.I]).to_list(), 0.3),\n", - " (QubitPauliString(qubits, [Pauli.Z, Pauli.Y, Pauli.X, Pauli.Y]).to_list(), 0.6),\n", - "]\n", - "inputs = {\n", - " \"ansatz\": build_ansatz().to_dict(),\n", - " \"a\": 0.2,\n", - " \"b\": 0.55,\n", - " \"c\": 0.75,\n", - " \"ham\": hamiltonian,\n", - "}" - ] - }, - { - "cell_type": "markdown", - "id": "037d98b1", - "metadata": {}, - "source": [ - "before we can run the simulation:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "53446615", - "metadata": {}, - "outputs": [], - "source": [ - "from tierkreis.controller import run_graph\n", - "from tierkreis.storage import read_outputs\n", - "\n", - "\n", - "storage.clean_graph_files()\n", - "run_graph(\n", - " storage,\n", - " multi_executor,\n", - " simulation_graph,\n", - " inputs,\n", - " polling_interval_seconds=0.2,\n", - ")\n", - "output = read_outputs(simulation_graph, storage)\n", - "print(output)" - ] - } - ], - "metadata": { - "execution": { - "timeout": -1 - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.13.11" - } + "cells": [ + { + "cell_type": "markdown", + "id": "d56e6c6d", + "metadata": {}, + "source": [ + "# Lesson 4: Putting everything together\n", + "\n", + "In this example we're going to apply the previously learned concept to run a Hamiltonian simulation:\n", + "1. Defining an ansatz using a `pytket` symbolic circuit\n", + "2. Provided a workflow to calculate an expected value of a circuit given a Pauli String:\n", + " - Using the symbolic substituiton from [Lesson 2](pytket_graph.ipynb)\n", + " - Using the `tkr-pytket-worker` for compilation and `tkr-aer-worker` for simulation similar to [Lesson 3](storage_and_executors.ipynb)\n", + "\n", + "3. Construct a workflow using input and output definitions, as well as builtin functionalities as shown in [Lesson 1](first_graph.ipynb)\n", + " - *(new)* Using higher order constructs like `map` to parallelize execution\n", + " - *(new)* Introduce the `fold` pattern\n", + "\n", + "## The ansatz\n", + "\n", + "We start by defining a symbolic circuit as an ansatz which will be the input to our workflow.\n", + "Were going to use a very simple circuit, which only uses 4 qubits and 3 symbols to match what we build so far.\n", + "Note that it doesn't have any measurements yet. We will add them in the next step when calculating the expected value." + ] }, - "nbformat": 4, - "nbformat_minor": 5 + { + "cell_type": "code", + "execution_count": null, + "id": "bfbaa4df", + "metadata": {}, + "outputs": [], + "source": [ + "from pytket.circuit import Circuit, fresh_symbol\n", + "\n", + "\n", + "def build_ansatz() -> Circuit:\n", + " a = fresh_symbol(\"a\")\n", + " b = fresh_symbol(\"b\")\n", + " c = fresh_symbol(\"c\")\n", + " circ = Circuit(4)\n", + " circ.CX(0, 1)\n", + " circ.CX(1, 2)\n", + " circ.CX(2, 3)\n", + " circ.Rz(a, 3)\n", + " circ.CX(2, 3)\n", + " circ.CX(1, 2)\n", + " circ.CX(0, 1)\n", + " circ.Rz(b, 0)\n", + " circ.CX(0, 1)\n", + " circ.CX(1, 2)\n", + " circ.CX(2, 3)\n", + " circ.Rz(c, 3)\n", + " circ.CX(2, 3)\n", + " circ.CX(1, 2)\n", + " circ.CX(0, 1)\n", + " return circ" + ] + }, + { + "cell_type": "markdown", + "id": "ca9d6bc5", + "metadata": {}, + "source": [ + "## Calculating expected values\n", + "\n", + "We will evaluate this circuit with an observable based on a Pauli string.\n", + "We want to measure the circuit according the Pauli string after which we can compile and submit it.\n", + "Using the respective workers we can construct a subgraph.\n", + "In Tierkreis graphs are first class citizens, which we can reuse in multiple locations.\n", + "In this example, we later want to map over this procedure with different parameters.\n", + "Defining the graph which taktes a circuit, a Pauli string and the number of shots." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0b051d3a", + "metadata": {}, + "outputs": [], + "source": [ + "from typing import NamedTuple, Literal\n", + "from tierkreis.builder import GraphBuilder\n", + "from tierkreis.controller.data.models import TKR\n", + "\n", + "from pytket_worker import (\n", + " append_pauli_measurement_impl,\n", + " optimise_phase_gadgets,\n", + " expectation,\n", + ")\n", + "from aer_worker import submit_single\n", + "\n", + "\n", + "class SubmitInputs(NamedTuple):\n", + " circuit: TKR[Literal[\"pytket._tket.circuit.Circuit\"]]\n", + " pauli_string: TKR[Literal[\"pytket._tket.pauli.QubitPauliString\"]]\n", + " n_shots: TKR[int]\n", + "\n", + "\n", + "def exp_val():\n", + " g = GraphBuilder(SubmitInputs, TKR[float])\n", + "\n", + " circuit = g.inputs.circuit\n", + " pauli_string = g.inputs.pauli_string\n", + " n_shots = g.inputs.n_shots\n", + "\n", + " measurement_circuit = g.task(append_pauli_measurement_impl(circuit, pauli_string))\n", + "\n", + " compiled_circuit = g.task(optimise_phase_gadgets(measurement_circuit))\n", + "\n", + " backend_result = g.task(submit_single(compiled_circuit, n_shots))\n", + " av = g.task(expectation(backend_result))\n", + " g.outputs(av)\n", + " return g" + ] + }, + { + "cell_type": "markdown", + "id": "d1993864", + "metadata": {}, + "source": [ + "## Building the workflow\n", + "\n", + "We are going to simulate a Hamiltonian given as a list of Pauli strings and their weights.\n", + "As additional inputs we take the parameters to our ansazt.\n", + "We can already build the first step of the graph, using the substitution worker we build before (see [Lesson 2](pytket_graph.ipynb))." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f5d2e7be", + "metadata": {}, + "outputs": [], + "source": [ + "from pytket_example_worker import substitute\n", + "\n", + "\n", + "class SymbolicExecutionInputs(NamedTuple):\n", + " a: TKR[float]\n", + " b: TKR[float]\n", + " c: TKR[float]\n", + " ham: TKR[list[tuple[Literal[\"pytket._tket.pauli.QubitPauliString\"], float]]]\n", + " ansatz: TKR[Literal[\"pytket._tket.circuit.Circuit\"]]\n", + "\n", + "\n", + "simulation_graph = GraphBuilder(SymbolicExecutionInputs, TKR[float])\n", + "substituted_circuit = simulation_graph.task(\n", + " substitute(\n", + " simulation_graph.inputs.ansatz,\n", + " simulation_graph.inputs.a,\n", + " simulation_graph.inputs.b,\n", + " simulation_graph.inputs.c,\n", + " )\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "c7f5f9a1", + "metadata": {}, + "source": [ + "### Using map \n", + "\n", + "Since the `exp_val()` workflow can run independently for each Pauli string we can implement this using a [map](../graphs/map.md) node.\n", + "Map nodes execute a subworkflow in parallel for an arbitrary number of different inputs.\n", + "\n", + "The node itself expects `Graphbuilder` instance for the workflow and a list of input parameters to map over.\n", + "In our case, the `exp_val()` workflow expects `SubmitInputs` where the field `circuit` and `n_shots` are fixed.\n", + "So it is sufficient to map over the pauli strings\n", + "We need to prepare the inputs accordingly, in this scenario we build a list of `SubmitInputs` each with Pauli string using a `lambda`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "021e0e5b", + "metadata": {}, + "outputs": [], + "source": [ + "from tierkreis.builtins import unzip\n", + "from tierkreis.controller.data.models import TKR\n", + "\n", + "pauli_strings_list, parameters_list = simulation_graph.task(\n", + " unzip(simulation_graph.inputs.ham)\n", + ")\n", + "input_circuits = simulation_graph.map(\n", + " lambda x: SubmitInputs(substituted_circuit, x, simulation_graph.const(100)),\n", + " pauli_strings_list,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "f452595f", + "metadata": {}, + "source": [ + "which we than can then supply to the map node:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bc10a674", + "metadata": {}, + "outputs": [], + "source": [ + "exp_values = simulation_graph.map(exp_val(), input_circuits)" + ] + }, + { + "cell_type": "markdown", + "id": "7788777e", + "metadata": {}, + "source": [ + "### Using the fold pattern\n", + "\n", + "To estimate the energy, we can take a weighted sum of the expectation values.\n", + "For this we want to compute a reduction of $f:\\mathbb{R}^2\\times\\mathbb{R} \\rightarrow \\mathbb{R} \\quad ((x,y), z) \\rightarrow x*y+z$ which we implement as a fold function.\n", + "This is a very common pattern; In Tierkreis the function $f$ will be implemented as a workflow using the functionality in `tierkreis.graphs.fold`.\n", + "\n", + "Defining $f$ as `compute_terms()` using builtins:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cd24e08c", + "metadata": {}, + "outputs": [], + "source": [ + "from tierkreis.builtins import add, times, untuple\n", + "from tierkreis.graphs.fold import FoldFunctionInput\n", + "\n", + "ComputeTermsInputs = FoldFunctionInput[\n", + " tuple[float, float], float\n", + "] # (value: (x,y) , accum: z) -> new_accum\n", + "\n", + "\n", + "def compute_terms():\n", + " g = GraphBuilder(ComputeTermsInputs, TKR[float])\n", + "\n", + " res_0, res_1 = g.task(untuple(g.inputs.value))\n", + " prod = g.task(times(res_0, res_1))\n", + " sum = g.task(add(g.inputs.accum, prod))\n", + "\n", + " g.outputs(sum)\n", + " return g" + ] + }, + { + "cell_type": "markdown", + "id": "af57c8ac", + "metadata": {}, + "source": [ + "Preparing the inputs for the fold graph we are going to use tuples `(x=exp_val, y=weight)` and defining the start value $ z=0 $:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9bab0de3", + "metadata": {}, + "outputs": [], + "source": [ + "from tierkreis.builtins import tkr_zip\n", + "from tierkreis.graphs.fold import FoldGraphInputs, fold_graph\n", + "\n", + "tuple_values = simulation_graph.task(tkr_zip(exp_values, parameters_list))\n", + "fold_inputs = FoldGraphInputs(simulation_graph.const(0.0), tuple_values)" + ] + }, + { + "cell_type": "markdown", + "id": "da997ea2", + "metadata": {}, + "source": [ + "applying the fold operation yields the final output" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d7c6fa18", + "metadata": {}, + "outputs": [], + "source": [ + "computed = simulation_graph.eval(fold_graph(compute_terms()), fold_inputs)\n", + "simulation_graph.outputs(computed)" + ] + }, + { + "cell_type": "markdown", + "id": "5999c0a5", + "metadata": {}, + "source": [ + "## Running the workflow \n", + "\n", + "As before we now have to set up tierkreis storage and executors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aec32e19", + "metadata": {}, + "outputs": [], + "source": [ + "from uuid import UUID\n", + "\n", + "from tierkreis.storage import FileStorage\n", + "from tierkreis.executor import ShellExecutor\n", + "\n", + "storage = FileStorage(UUID(int=102), name=\"hamiltonian\")\n", + "example_executor = ShellExecutor(None, workflow_dir=storage.workflow_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "20848215", + "metadata": {}, + "source": [ + "and provide the inputs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2d3b86c0", + "metadata": {}, + "outputs": [], + "source": [ + "from pytket._tket.unit_id import Qubit\n", + "from pytket.pauli import Pauli, QubitPauliString\n", + "\n", + "qubits = [Qubit(0), Qubit(1), Qubit(2), Qubit(3)]\n", + "hamiltonian = [\n", + " (QubitPauliString(qubits, [Pauli.X, Pauli.Y, Pauli.X, Pauli.I]).to_list(), 0.1),\n", + " (QubitPauliString(qubits, [Pauli.Y, Pauli.Z, Pauli.X, Pauli.Z]).to_list(), 0.5),\n", + " (QubitPauliString(qubits, [Pauli.X, Pauli.Y, Pauli.Z, Pauli.I]).to_list(), 0.3),\n", + " (QubitPauliString(qubits, [Pauli.Z, Pauli.Y, Pauli.X, Pauli.Y]).to_list(), 0.6),\n", + "]\n", + "inputs = {\n", + " \"ansatz\": build_ansatz().to_dict(),\n", + " \"a\": 0.2,\n", + " \"b\": 0.55,\n", + " \"c\": 0.75,\n", + " \"ham\": hamiltonian,\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "037d98b1", + "metadata": {}, + "source": [ + "before we can run the simulation:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "53446615", + "metadata": {}, + "outputs": [], + "source": [ + "from tierkreis.controller import run_graph\n", + "from tierkreis.storage import read_outputs\n", + "\n", + "\n", + "storage.clean_graph_files()\n", + "run_graph(\n", + " storage,\n", + " example_executor,\n", + " simulation_graph,\n", + " inputs,\n", + " polling_interval_seconds=0.1,\n", + ")\n", + "output = read_outputs(simulation_graph, storage)\n", + "print(output)" + ] + } + ], + "metadata": { + "execution": { + "timeout": -1 + }, + "kernelspec": { + "display_name": "tierkreis", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 } diff --git a/docs/source/examples/hello_world_graph.ipynb b/docs/source/examples/hello_world_graph.ipynb index 0281e4640..0e2834749 100644 --- a/docs/source/examples/hello_world_graph.ipynb +++ b/docs/source/examples/hello_world_graph.ipynb @@ -5,51 +5,238 @@ "id": "4035de03", "metadata": {}, "source": [ - "# Hello World: Tierkreis Edition\n", - "In this example we will construct a workflow graph that uses the `greet` function from the `hello_world_worker` we defined in the previous example.\n", - "We want to write a graph that produces the output `hello world`.\n", - "First we provide the necessary imports, the `GraphBuilder`, a class for constructing workflows, and `TKR`; you can think of it as tierkreis types (more on this at the bottom of the page). \n", - "This ensures, that only types are used that can be processed internally." + "# Hello World revisited: writing workers\n", + "\n", + "In this example we will construct our own worker and rewrite the previous example to use it instead of the builtins.\n", + "We will first write the `hello_world_worker` with a `greet` function that acts similar to `concat`.\n", + "\n", + "## Writing tierkreis workers\n", + "\n", + "One of the core concepts in tierkreis are workers.\n", + "In this example, we will cover the conceptual ideas of workers and how to write one.\n", + "For this we will write a simple worker from scratch.\n", + "The `tkr init` cli provides a convenient set up for this which will automate most of the tasks described after.\n", + "If you followed the previous tutorial, you can also update the example worker to your liking.\n", + "\n", + "```{note}\n", + "As explained in the previous section, workers are independent projects.\n", + "The worker we are going to implement already exists in the project, so that we can use its package name.\n", + "If you're using the `example_worker` from the previous tutorial, make sure to change `hello_world_worker` to `example_worker`.\n", + "If you don't want to set up the worker yourself, you can clone the [Tierkreis project](https://github.com/Quantinuum/tierkreis) and use the worker from there.\n", + "To have python recognize the package you have to run `uv sync` once.\n", + "```\n", + "\n", + "### Concepts\n", + "\n", + "Workers encapsulate a set of functionalities which are described in their API (stubs).\n", + "The API contains a list of typed function interfaces which are implemented by the worker.\n", + "Tierkreis expects the functions to be stateless and will only checkpoint inputs and outputs of the function call.\n", + "Although, it is still possible to introduce state, but this then has to be managed by the programmer.\n", + "\n", + "Each worker is an independent program, can have multiple implementations (e.g. based on architecture) of these interfaces and has its own dependencies.\n", + "There are two main ways to write workers:\n", + "- Using the tierkreis `Worker` class for python based workers\n", + "- Using a spec file to define the interface for non-python workers\n", + "\n", + "In this example, we will write a simple worker, that greets a subject.\n", + "`greet: str -> str -> str `\n", + "\n", + "### Worker Setup\n", + "\n", + "A worker consist of four conceptual parts which we will set up in this example.\n", + "1. Dependency information. For python workers, we recommend using `uv` and a `pyproject.toml`.\n", + "2. A main entry point into the worker. For python workers this is `main.py`. Alternatively any runnable binary will work; Tierkreis sets this up when using the cli.\n", + "3. The api definition of the worker typically called `stubs.py` or `api.py`. For python workers this can be generated from the main file. For other workers you can use a typespec file.\n", + "4. (Optional) Library code containing the core logic.\n", + "\n", + "To set this up, you can run:\n", + "\n", + "```bash\n", + "tkr init worker --worker-name hello_world_worker\n", + "```\n", + "\n", + "### 1. Dependencies\n", + "\n", + "Dependency information should be provide, such that the worker can run in its installed location.\n", + "For python, this is canonically done through the `pyproject.toml` which tierkreis also uses.\n", + "A minimal example contains\n", + "\n", + "```toml\n", + "[project]\n", + "name = \"hello-world-worker\"\n", + "version = \"0.1.0\"\n", + "requires-python = \">=3.12\"\n", + "dependencies = [\"tierkreis\"]\n", + "\n", + "[dependency-groups]\n", + "# Optional dev dependency group\n", + "dev = [\"ruff\"]\n", + "```\n", + "Ruff is an optional dependency to format the code later.\n", + "\n", + "### 2. Worker entry point\n", + "\n", + "The entrypoint contains the implementations of the interface.\n", + "We are going to use the build in `Worker` class to define the worker.\n", + "Each worker needs a unique name, here we chose `hello_world_worker`.\n", + "In tierkreis the worker is defined by its name and the directory it lives in.\n", + "\n", + "In `tkr_hello_world_worker_impl/impl.py` we define the worker functionality by instantiating a worker" ] }, { "cell_type": "code", "execution_count": null, - "id": "98c7c0cd", + "id": "cc88bb3e", "metadata": {}, "outputs": [], "source": [ - "%pip install tierkreis" + "import logging\n", + "\n", + "from tierkreis import Worker\n", + "\n", + "logger = logging.getLogger(__name__)\n", + "worker = Worker(\"hello_world_worker\")" + ] + }, + { + "cell_type": "markdown", + "id": "af3d9b07", + "metadata": {}, + "source": [ + "From here we can add functions to the worker by decorating python functions with `@worker.task()`.\n", + "To make use of the type safety in tierkreis, you should provide narrow types to the function." ] }, { "cell_type": "code", "execution_count": null, - "id": "d57959b4", + "id": "2d89f49c", "metadata": {}, "outputs": [], "source": [ - "from tierkreis.builder import GraphBuilder\n", - "from tierkreis.controller.data.models import TKR" + "@worker.task()\n", + "def greet(greeting: str, subject: str) -> str:\n", + " logger.info(\"%s %s\", greeting, subject)\n", + " return greeting + subject" + ] + }, + { + "cell_type": "markdown", + "id": "eec03dca", + "metadata": {}, + "source": [ + "Now we have defined a greet function that concatenates a two strings, similar to the `concat` builtin.\n", + "It now lives in the namespace `hello_world_worker.greet`.\n", + "To make sure the worker runs correctly we need to add the following lines in `main.py` which is usually autogenerated:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e407402c", + "metadata": {}, + "outputs": [], + "source": [ + "%%script false --no-raise-error\n", + "from sys import argv\n", + "\n", + "from tkr_hello_world_worker_impl import worker \n", + "\n", + "def main():\n", + " worker.app(argv)\n", + "\n", + "\n", + "if __name__ == \"__main__\":\n", + " main()" ] }, { "cell_type": "markdown", - "id": "f62ea127", + "id": "c9e9b4e4", "metadata": {}, "source": [ - "A workflow is just a representation of a computation.\n", - "The example we are going to build represents a function `str -> str`.\n", - "This signature is defined at instantiation of the Graph builder object" + "They make sure that tierkreis can call the worker correctly.\n", + "\n", + "### 3. Api definition\n", + "\n", + "For python workers, tierkreis can automatically generate the api definition.\n", + "You can either run the app as `uv run main.py --stubs-path ../api/api.py` or call the function directly." ] }, { "cell_type": "code", "execution_count": null, - "id": "9224b7d7", + "id": "e57e1c4c", "metadata": {}, "outputs": [], "source": [ + "%%script false --no-raise-error\n", + "from pathlib import Path\n", + "worker.namespace.write_stubs(Path(\"../api/api.py\"))" + ] + }, + { + "cell_type": "markdown", + "id": "7307f8d8", + "metadata": {}, + "source": [ + "This will generate a file `api.py` that contains the interface definition that can now be used as `task` in a Tierkreis workflow.\n", + "The code will look like this:\n", + "```python\n", + "\"\"\"Code generated from hello_world_worker namespace. Please do not edit.\"\"\"\n", + "\n", + "from typing import NamedTuple\n", + "from tierkreis.controller.data.models import TKR\n", + "\n", + "\n", + "class greet(NamedTuple):\n", + " greeting: TKR[str] # noqa: F821 # fmt: skip\n", + " subject: TKR[str] # noqa: F821 # fmt: skip\n", + "\n", + " @staticmethod\n", + " def out() -> type[TKR[str]]: # noqa: F821 # fmt: skip\n", + " return TKR[str] # noqa: F821 # fmt: skip\n", + "\n", + " @property\n", + " def namespace(self) -> str:\n", + " return \"hello_world_worker\"\n", + "\n", + "```\n", + "Which defines a class describing the function.\n", + "\n", + "### 4. Using an existing library\n", + "\n", + "For simple workers we can directly write our code as python functions.\n", + "If you already have a library we recommend keeping the structure, having a single file for your worker tasks.\n", + "You can import you library functions there and wrap them with a small function, e.g.:\n", + "```python\n", + "from my_lib import my_func\n", + "\n", + "@worker.task()\n", + "def my_func_wrapper() -> None:\n", + " my_func()\n", + "\n", + "```\n", + "\n", + "## Writing the workflow\n", + "\n", + "\n", + "We want to write a workflow that produces the output `Hello `, which is similar to the one from the initial example.\n", + "But instead of the builtins, we're going to use the newly defined tasks.\n", + "First we provide the necessary imports, and declare the `GraphBuilder`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d57959b4", + "metadata": {}, + "outputs": [], + "source": [ + "from tierkreis.builder import GraphBuilder\n", + "from tierkreis.controller.data.models import TKR\n", + "\n", "graph = GraphBuilder(inputs_type=TKR[str], outputs_type=TKR[str])" ] }, @@ -58,8 +245,7 @@ "id": "9b76964b", "metadata": {}, "source": [ - "We could simply use a graph that just returns its input.\n", - "Instead were going to construct the string `hello world` from an constant value and an input.\n", + "As before were going to construct the string `Hello world` from an constant value and a user input.\n", "First we define the constant part, by adding a constant node to the graph." ] }, @@ -81,8 +267,12 @@ "We capture the output of that node in the variable `hello`, which we can use as input to other nodes to insert an edge in the graph.\n", "Inputs to the graph cah be referenced in two ways depending if we have a single input (`graph.inputs`) or multiple inputs (`graph.inputs.`).\n", "We will cover multiple inputs in a later example.\n", - "For now, we want to call the `greet` function from the `hello_world_worker`.\n", - "To use the type hints from that worker we can import the interface and then use it as a task." + "For now, we want to call the `greet` function from the `hello_world_worker` we defined early.\n", + "To use the type hints from that worker we can import the interface and then use it as a task.\n", + "```{important}\n", + "During definition, we use the **api** which means we don't need to have the worker available in our project.\n", + "During runtime, an executor will invoke the workers entry point (`main.py`).\n", + "```" ] }, { @@ -92,7 +282,7 @@ "metadata": {}, "outputs": [], "source": [ - "from hello_world_worker import greet\n", + "from hello_world_worker import greet # noqa: F811\n", "\n", "output = graph.task(greet(greeting=hello, subject=graph.inputs))" ] @@ -130,38 +320,32 @@ "- A dictionary providing the inputs to the graph. A single input will always have the key `\"value\"`. In this case we want to set it to `{\"value\": \"world!\"}`\n", "\n", "To make this reproducible we can define a `name` and a `run_id`.\n", - "To make sure tierkreis can find our worker, we have to define the `registry_path`.\n", - "Since it is a python worker we use uv to run it." + "\n", + "In contrast to before, we now also have to define an executor that will find our worker.\n", + "In `run_workflow` we have to define the `registry_path` to the location that contains the worker directory.\n", + "We point it to the parent directory of the `hello_world_worker`." ] }, { "cell_type": "code", "execution_count": null, - "id": "20f07fa0", + "id": "10c39de1", "metadata": {}, "outputs": [], "source": [ "from pathlib import Path\n", "\n", - "from tierkreis.cli.run_workflow import run_workflow\n", - "\n", + "registry_path = (\n", + " Path().parent / \"example_workers\"\n", + ") # Look for workers in the `example_workers` directory.\n", "\n", - "def main(input_value: str) -> None:\n", - " run_workflow(\n", - " graph.data,\n", - " {\"value\": input_value},\n", - " name=\"hello_world\",\n", - " run_id=100, # Assign a fixed uuid for our workflow.\n", - " registry_path=Path().parent\n", - " / \"example_workers\", # Look for workers in the `example_workers` directory.\n", - " use_uv_executor=True,\n", - " print_output=True,\n", - " )" + "# registry_path = Path().parent / \"workers\"\n", + "# TODO: uncomment this if you're using the init example" ] }, { "cell_type": "markdown", - "id": "4488c5ea", + "id": "63321b1d", "metadata": {}, "source": [ "Finally we can run the code, which will print the desired output `Hello world!`." @@ -170,25 +354,22 @@ { "cell_type": "code", "execution_count": null, - "id": "2689da35", + "id": "20f07fa0", "metadata": {}, "outputs": [], "source": [ - "if __name__ == \"__main__\":\n", - " main(\"world!\")" - ] - }, - { - "cell_type": "markdown", - "id": "84ab8e63", - "metadata": {}, - "source": [ - "## On tierkreis types\n", + "from tierkreis.cli.run_workflow import run_workflow\n", "\n", - "Tierkreis values correspond to the edges in the graph.\n", - "These values can have a type assigned to them at construction time.\n", - "The set of available types is a subset of all python types, e.g. we require serialization; see more in [complex types](../worker/complex_types.md) how to add your own serialization.\n", - "As result we use the `TKR` container to promote the python types into tierkreis compatible types." + "input_value = \" world!\" # TODO change this\n", + "run_workflow(\n", + " graph.data,\n", + " {\"value\": input_value},\n", + " name=\"hello_world\",\n", + " run_id=100, # Assign a fixed uuid for our workflow.\n", + " registry_path=registry_path,\n", + " use_uv_executor=True,\n", + " print_output=True,\n", + ")" ] } ], diff --git a/docs/source/examples/hpc.ipynb b/docs/source/examples/hpc.ipynb index ff7a20e69..9264e7e30 100644 --- a/docs/source/examples/hpc.ipynb +++ b/docs/source/examples/hpc.ipynb @@ -14,16 +14,6 @@ "As before were defining inputs and outputs:" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "3c07e149", - "metadata": {}, - "outputs": [], - "source": [ - "%pip install tierkreis pytket" - ] - }, { "cell_type": "code", "execution_count": null, @@ -64,12 +54,12 @@ "source": [ "from tierkreis.builder import GraphBuilder\n", "from substitution_worker import substitute\n", - "from tierkreis.pytket_worker import (\n", + "from pytket_worker import (\n", " add_measure_all,\n", " expectation,\n", " optimise_phase_gadgets,\n", ")\n", - "from tierkreis.aer_worker import submit_single\n", + "from aer_worker import submit_single\n", "\n", "\n", "def symbolic_execution() -> GraphBuilder:\n", @@ -293,7 +283,7 @@ "run_graph(\n", " storage,\n", " multi_executor,\n", - " symbolic_execution().data,\n", + " symbolic_execution(),\n", " {\n", " \"ansatz\": build_ansatz(),\n", " \"a\": 0.2,\n", @@ -302,13 +292,13 @@ " },\n", " polling_interval_seconds=0.1,\n", ")\n", - "output = read_outputs(symbolic_execution().data, storage)" + "output = read_outputs(symbolic_execution(), storage)" ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "tierkreis", "language": "python", "name": "python3" }, diff --git a/docs/source/examples/index.md b/docs/source/examples/index.md deleted file mode 100644 index 5467ad8d0..000000000 --- a/docs/source/examples/index.md +++ /dev/null @@ -1,42 +0,0 @@ -# Examples - -This directory contains examples how to use Tierkreis to write workflows. -Before writing workflows, we will shortly recap how to write a worker. -It is intended that these notebooks are read in the following order. - -```{toctree} -:maxdepth: 1 -worker.ipynb -hello_world_graph.ipynb -storage_and_executors.ipynb -types_and_defaults.ipynb -multiple_outputs.ipynb -polling_and_dir.ipynb -parallelism.ipynb -errors_and_debugging.ipynb -restart.ipynb -signing_graph.ipynb -hpc.ipynb -scipy.ipynb -hamiltonian.ipynb -qsci.ipynb -``` - -## How to use - -In the given examples you will be developing code involving Tierkreis workers. -Whenever you see an import containing `*_worker` this means one of the workers will be invoked (except for the `builtin`s). -To ensure the examples will run correctly you will need to have the worker code available too. -The simplest way to set this up is to clone the entire repository before running any of the examples. - -``` -git clone https://github.com/Quantinuum/tierkreis.git -``` - -To set up the environment we us uv: - -``` -uv sync --all-extras -``` - -When running the notebooks select the kernel corresponding to the uv environment. diff --git a/docs/source/examples/parallelism.ipynb b/docs/source/examples/parallelism.ipynb index 066a7c403..32c1565fd 100644 --- a/docs/source/examples/parallelism.ipynb +++ b/docs/source/examples/parallelism.ipynb @@ -17,16 +17,6 @@ "2. Using the [qulacs](https://github.com/qulacs/qulacs) simulator" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "ee0cabe0", - "metadata": {}, - "outputs": [], - "source": [ - "%pip install tierkreis pytket qiskit-aer" - ] - }, { "cell_type": "code", "execution_count": null, @@ -38,11 +28,11 @@ "from tierkreis.builder import GraphBuilder\n", "from tierkreis.controller.data.models import TKR, OpaqueType\n", "from tierkreis.builtins import untuple\n", - "from tierkreis.aer_worker import (\n", + "from aer_worker import (\n", " get_compiled_circuit as aer_compile,\n", " run_circuit as aer_run,\n", ")\n", - "from tierkreis.qulacs_worker import (\n", + "from qulacs_worker import (\n", " get_compiled_circuit as qulacs_compile,\n", " run_circuit as qulacs_run,\n", ")\n", @@ -335,7 +325,7 @@ "timeout": 120 }, "kernelspec": { - "display_name": "Python 3", + "display_name": "tierkreis", "language": "python", "name": "python3" }, diff --git a/docs/source/examples/polling_and_dir.ipynb b/docs/source/examples/polling_and_dir.ipynb index 97ccd00de..483693288 100644 --- a/docs/source/examples/polling_and_dir.ipynb +++ b/docs/source/examples/polling_and_dir.ipynb @@ -13,16 +13,6 @@ "This can be configured in the storage by providing the `tierkreis_directory` argument to the storage layer (e.g. `checkpoints2`).\n" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "052a8990", - "metadata": {}, - "outputs": [], - "source": [ - "%pip install tierkreis qnexus" - ] - }, { "cell_type": "code", "execution_count": null, @@ -163,7 +153,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "tierkreis", "language": "python", "name": "python3" }, diff --git a/docs/source/examples/pytket_graph.ipynb b/docs/source/examples/pytket_graph.ipynb new file mode 100644 index 000000000..fd1f1ca70 --- /dev/null +++ b/docs/source/examples/pytket_graph.ipynb @@ -0,0 +1,299 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c0439942", + "metadata": {}, + "source": [ + "# Lesson 2: Writing Workers\n", + "\n", + "In the previous example you wrote a worker only using Tierkreis base functionality.\n", + "One of the main benefits of using Tierkreis is that you can easily transform existing code and gain the benefits of Tierkeis, e.g., checkpointing and repeteability.\n", + "\n", + "In this example we're going to look at a common task and transform it into a graph.\n", + "We're going to use [pytket](https://docs.quantinuum.com/tket/api-docs/) to:\n", + "1. Define a symbolic circuit\n", + "2. Substitute its symbolic parameters at runtime\n", + "3. Compile and execute the circuit\n", + "\n", + "\n", + "## Prerequisite\n", + "\n", + "If you haven't done so, set up a Tierkreis project and install pytket\n", + "\n", + "```bash\n", + "uv init\n", + "uv add tierkreis pytket pytket-qiskit sympy\n", + "uv run tkr project init\n", + "```\n", + "\n", + "## Setting up the worker\n", + "\n", + "Using the the Tierkreis cli is the easiest the way to set up a worker:\n", + "\n", + "```bash\n", + "uv run tkr init worker -n pytket_example_worker\n", + "uv sync --all-extras\n", + "```\n", + "\n", + "Which will set up the packages in a convenient way for you.\n", + "```{info}\n", + "If you don't want to use the cli, you need a way to provide the worker API to your graph construction.\n", + "You can find more information [here](../worker/index.md)\n", + "```\n", + "\n", + "There are two files which you need to care about which are:\n", + "- `impl.py` here you will implement your workers functionality\n", + "- `api.py` is an autogenerated file which contains the task definitions we will use in the graph.\n", + "\n", + "In `impl.py` you will find the following:\n", + "\n", + "```python\n", + "worker = Worker(\"pytket_example_worker\")\n", + "\n", + "@worker.task()\n", + "def your_worker_task(value: int) -> int:\n", + " return value\n", + "```\n", + "\n", + "As you can see, generating a task for Tierkeis simply means adding `@worker.task()` to a function.\n", + "We're going to use the same pattern to expose `pytket`s functionality to in our new worker.\n", + "\n", + "## Defining the tasks\n", + "\n", + "For the following assume we're going to use a simple quantum circuits with three symbols $A,B,C$.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16ecff0a", + "metadata": {}, + "outputs": [], + "source": [ + "from pytket.circuit import Circuit, fresh_symbol\n", + "\n", + "\n", + "a = fresh_symbol(\"a\")\n", + "b = fresh_symbol(\"b\")\n", + "c = fresh_symbol(\"c\")\n", + "circ = Circuit(3)\n", + "circ.Rz(a, 0)\n", + "circ.Rz(b, 0)\n", + "circ.Rz(c, 0)\n", + "circ.measure_all()" + ] + }, + { + "cell_type": "markdown", + "id": "1416b27e", + "metadata": {}, + "source": [ + "Using plain `pytket` you could know substitute the symbols like so:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2234b31f", + "metadata": {}, + "outputs": [], + "source": [ + "from sympy import Symbol\n", + "\n", + "circ.symbol_substitution({Symbol(\"a\"): -1, Symbol(\"b\"): 0, Symbol(\"c\"): 1})" + ] + }, + { + "cell_type": "markdown", + "id": "01383a58", + "metadata": {}, + "source": [ + "writing this as a worker task means wrapping it with a task function.\n", + "```{important}\n", + "Use type hints so that Tierkreis can validate the task during construction.\n", + "Since values are represented by edges, we need a return value, its not sufficient to mutate the state.\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19da7c1c", + "metadata": {}, + "outputs": [], + "source": [ + "# This is added for validity of the example\n", + "from tierkreis import Worker\n", + "\n", + "worker = Worker(\"pytket_example_worker\")\n", + "\n", + "\n", + "@worker.task()\n", + "def substitute(circuit: Circuit, a: float, b: float, c: float) -> Circuit:\n", + " circuit.symbol_substitution({Symbol(\"a\"): a, Symbol(\"b\"): b, Symbol(\"c\"): c})\n", + " return circuit" + ] + }, + { + "cell_type": "markdown", + "id": "ad3bf915", + "metadata": {}, + "source": [ + "Similarly we can now define other tasks using `pytket` and and `@worker.task()`:\n", + "\n", + "For compilation, e.g., optimizing phase gadgets:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f8d5afba", + "metadata": {}, + "outputs": [], + "source": [ + "from pytket.transform import Transform\n", + "\n", + "\n", + "@worker.task()\n", + "def optimise(circuit: Circuit) -> Circuit:\n", + "\n", + " Transform.OptimisePhaseGadgets().apply(circuit)\n", + " return circuit" + ] + }, + { + "cell_type": "markdown", + "id": "cd2dcc15", + "metadata": {}, + "source": [ + "And simulation, e.g. using on an aer simulator using `pytket-qiskit`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "468d3c6f", + "metadata": {}, + "outputs": [], + "source": [ + "from pytket.backends.backendresult import BackendResult\n", + "from pytket.extensions.qiskit import AerBackend\n", + "\n", + "\n", + "@worker.task()\n", + "def simulate(circuit: Circuit) -> BackendResult:\n", + " backend = AerBackend()\n", + " return backend.run_circuit(circuit, n_shots=1000)" + ] + }, + { + "cell_type": "markdown", + "id": "4bb179a7", + "metadata": {}, + "source": [ + "\n", + "## Generating stubs\n", + "\n", + "To generate the APIs for all workers, you can use the cli with\n", + "```bash\n", + "uv run tkr init stubs\n", + "```\n", + "\n", + "Depending on your development environment it might be necessary to resatart your language server or `uv sync --all-extras` to pick up the update changes.\n", + "\n", + "\n", + "### Opaque Types\n", + "Tierkreis can use any type that is serializable as in and outputs, e.g., the `Circuit` type from `pytket` library.\n", + "To make such types available without bleeding dependencies into graph code, Tiekreis wraps them as `OpaqueType` with a reference to the original implementation.\n", + "In this example the `circuit` inputs of the tasks would be\n", + "\n", + "```python\n", + "circuit: TKR[OpaqueType[\"pytket._tket.circuit.Circuit\"]] \n", + "```\n", + "\n", + "## Using the tasks\n", + "\n", + "Now you can use the newly declared tasks in a graph similar to how you used the `builtin` functionality.\n", + "You have to import the task API from the worker first which you then can use with a task node.\n", + "First we declare the graph" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fe7d6c0f", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "acf47c53", + "metadata": {}, + "outputs": [], + "source": [ + "from typing import NamedTuple\n", + "from tierkreis.builder import GraphBuilder\n", + "from tierkreis.controller.data.models import TKR\n", + "\n", + "\n", + "class PytketInputs(NamedTuple):\n", + " circuit: TKR[Circuit]\n", + " a: TKR[float]\n", + " b: TKR[float]\n", + " c: TKR[float]\n", + "\n", + "\n", + "graph = GraphBuilder(PytketInputs, TKR[BackendResult])" + ] + }, + { + "cell_type": "markdown", + "id": "c16d4459", + "metadata": {}, + "source": [ + "and then add the tasks:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e5e445a1", + "metadata": {}, + "outputs": [], + "source": [ + "from pytket_example_worker import substitute, optimise, simulate # noqa: F811\n", + "\n", + "substituted = graph.task(\n", + " substitute(graph.inputs.circuit, graph.inputs.a, graph.inputs.a, graph.inputs.a)\n", + ")\n", + "optimized = graph.task(optimise(substituted))\n", + "result = graph.task(simulate(optimized))\n", + "graph.outputs(result)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "tierkreis", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/source/examples/qsci.ipynb b/docs/source/examples/qsci.ipynb index 7a2f1b25f..f490cc807 100644 --- a/docs/source/examples/qsci.ipynb +++ b/docs/source/examples/qsci.ipynb @@ -33,16 +33,6 @@ "First we will import some helper types used in the worker:" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "8c9b8dd0", - "metadata": {}, - "outputs": [], - "source": [ - "%pip install tierkreis pytket" - ] - }, { "cell_type": "code", "execution_count": null, @@ -342,10 +332,10 @@ "metadata": {}, "outputs": [], "source": [ - "from tierkreis.aer_worker import submit_single\n", + "from aer_worker import submit_single\n", "from tierkreis.builder import GraphBuilder\n", "from tierkreis.controller.data.models import TKR, OpaqueType\n", - "from tierkreis.quantinuum_worker import compile_circuit_quantinuum\n", + "from quantinuum_worker import compile_circuit_quantinuum\n", "\n", "\n", "def _compile_and_run() -> GraphBuilder[\n", @@ -565,7 +555,7 @@ "timeout": -1 }, "kernelspec": { - "display_name": "Python 3", + "display_name": "tierkreis", "language": "python", "name": "python3" }, diff --git a/docs/source/examples/scipy.ipynb b/docs/source/examples/scipy.ipynb index 334ff5cb6..57a8217d9 100644 --- a/docs/source/examples/scipy.ipynb +++ b/docs/source/examples/scipy.ipynb @@ -17,16 +17,6 @@ "We use this in a simple graph simple graph." ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "d24594b1", - "metadata": {}, - "outputs": [], - "source": [ - "%pip install tierkreis numpy scipy" - ] - }, { "cell_type": "code", "execution_count": null, @@ -190,7 +180,7 @@ }, "outputs": [], "source": [ - "cd ~/.tierkreis/checkpoints/00000000-0000-0000-0000-0000000000cf`\n", + "cd ~/.tierkreis/checkpoints/00000000-0000-0000-0000-0000000000cf\n", "cat ./-.N2/outputs/value" ] }, @@ -219,7 +209,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "tierkreis", "language": "python", "name": "python3" }, diff --git a/docs/source/examples/signing_graph.ipynb b/docs/source/examples/signing_graph.ipynb index d0dfa2ea4..a2b4ef908 100644 --- a/docs/source/examples/signing_graph.ipynb +++ b/docs/source/examples/signing_graph.ipynb @@ -57,16 +57,6 @@ "Now we can generate the stubs using the cli `tkr init stubs` or from python." ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "631b6c35", - "metadata": {}, - "outputs": [], - "source": [ - "%pip install tierkreis" - ] - }, { "cell_type": "code", "execution_count": null, @@ -268,7 +258,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "tierkreis", "language": "python", "name": "python3" }, diff --git a/docs/source/examples/storage_and_executors.ipynb b/docs/source/examples/storage_and_executors.ipynb index 64c698a9c..1b83751f4 100644 --- a/docs/source/examples/storage_and_executors.ipynb +++ b/docs/source/examples/storage_and_executors.ipynb @@ -5,25 +5,16 @@ "id": "3dee6016", "metadata": {}, "source": [ - "# Configuring Storage and Executors, Submitting to Quantinuum, Using Predefined Workers\n", - "In this example we're covering a finer grained control over the workflow run.\n", - "These are the internals that are set inside the `run_workflow` function.\n", - "For this we're going to construct a graph thad runs a `pytket` circuit on a simulator using the `quantinuum_worker`\n", - "The worker can be installed using `pip install tkr-quantinuum-worker`\n", + "# Lesson 3: Using predefined workers\n", "\n", - "## Opaque Types\n", - "Tierkreis can use any type that is serializable as in and outputs.\n", - "To use such types for type hinting you can use the `OpaqueType`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f554cd69", - "metadata": {}, - "outputs": [], - "source": [ - "%pip install tierkreis pytket qnexus" + "In this example we're going beyoind writing our workers.\n", + "For this we're going to construct a graph that\n", + "- compiles a circuit using the `tkr-pytket-worker`\n", + "- run it on a simulator using the `pytket-qiskit` extension in the `tkr-aer-worker`\n", + "The workers can be installed using `uv add tkr-pytket-worker-impl tkr-aer-worker-impl`\n", + "\n", + "In this example we're also covering a finer grained control over the workflow run.\n", + "These are the internals that are set inside the `run_workflow` function." ] }, { @@ -44,10 +35,9 @@ "id": "ed257edc", "metadata": {}, "source": [ - "Now we can construct a graph using the `quantinuum_worker`.\n", - "The api definitions for workers build by the tierkreis teams are already included in tierkreis.\n", - "Still it is necessary to install the worker manually.\n", - "We define a graph `Circuit -> BackendResult` using hardcoded information for which emulator backend to use." + "Now we can construct a graph using the Tierkreis workers\n", + "The api definitions for workers can be imported from the `pytket_worker` (`aer_worker`) package which is a dependency we add when installing `tkr-quantinuum-worker-impl` (`tkr-aer-worker-impl`).\n", + "We define a graph `Circuit -> BackendResult` using a predefined compilation pass." ] }, { @@ -59,20 +49,17 @@ "source": [ "from tierkreis.builder import GraphBuilder\n", "from tierkreis.controller.data.models import TKR\n", - "from tierkreis.quantinuum_worker import (\n", - " compile_using_info,\n", - " get_backend_info,\n", - " run_circuit,\n", - ")\n", + "from pytket_worker import compile_generic_with_fixed_pass\n", + "from aer_worker import run_circuit\n", "\n", "g = GraphBuilder(TKR[Circuit], TKR[BackendResult])\n", - "info = g.task(get_backend_info(device_name=g.const(\"H2-1\")))\n", - "compiled_circuit = g.task(compile_using_info(g.inputs, info))\n", + "compiled_circuit = g.task(\n", + " compile_generic_with_fixed_pass(g.inputs, optimisation_level=g.const(2))\n", + ")\n", "results = g.task(\n", " run_circuit(\n", - " circuit=compiled_circuit,\n", + " circuit=compiled_circuit, # type: ignore\n", " n_shots=g.const(10),\n", - " device_name=g.const(\"H2-1SC\"),\n", " ),\n", ")\n", "g.outputs(results)" @@ -107,9 +94,12 @@ "id": "1a08a288", "metadata": {}, "source": [ - "Since the `quantinuum_worker` is a python worker we will use `uv` to run it.\n", "An executor lives in context, where it can access workers to run their `main` entrypoints.\n", - "We define this by providing a path to the directory our workers live in, in this case in the `tierkreis_workers` directory.\n" + "We define this by providing a path to the directory our workers live in, in this case in the `tierkreis_workers` directory.\n", + "\n", + "If you cloned the directory you now have the source of both workers.\n", + "You could run them using `uv run main.py`.\n", + "For this use case we have the `UvExecutor`." ] }, { @@ -127,26 +117,24 @@ }, { "cell_type": "markdown", - "id": "69f11464", + "id": "4bd2a05d", "metadata": {}, "source": [ - "Since this graph is using the `qnexus` api internally you also need to run the following once: " + "As an alternative (or if you only installed the worker), the worker exports a script `tkr-pyktet-worker`(`tkr-aer-worker`).\n", + "It functions as a shell binary, hence we can use the `ShellExecutor` for it.\n", + "Since using `uv add` will make this script available from within our python environment, we don't need to point to a directory." ] }, { "cell_type": "code", "execution_count": null, - "id": "901c2287", - "metadata": { - "tags": [ - "skip-execution" - ] - }, + "id": "3d234c7a", + "metadata": {}, "outputs": [], "source": [ - "from qnexus.client.auth import login\n", + "from tierkreis.executor import ShellExecutor\n", "\n", - "login()" + "executor = ShellExecutor(registry_path=None, workflow_dir=storage.workflow_dir)" ] }, { @@ -154,7 +142,7 @@ "id": "cc099589", "metadata": {}, "source": [ - "Once we provide the graph inputs we can now run it by providing a storage and an executor." + "Once we provide the workflow inputs we can now run it by providing a storage and an executor." ] }, { @@ -197,7 +185,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "tierkreis", "language": "python", "name": "python3" }, diff --git a/docs/source/examples/types_and_defaults.ipynb b/docs/source/examples/types_and_defaults.ipynb index 746244b67..8e3a55494 100644 --- a/docs/source/examples/types_and_defaults.ipynb +++ b/docs/source/examples/types_and_defaults.ipynb @@ -19,16 +19,6 @@ "Were building a graph circuit -> int -> (BackendResult, int)" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "def19061", - "metadata": {}, - "outputs": [], - "source": [ - "%pip install tierkreis pytket" - ] - }, { "cell_type": "code", "execution_count": null, @@ -67,7 +57,7 @@ "metadata": {}, "outputs": [], "source": [ - "from tierkreis.aer_worker import get_compiled_circuit, submit_single\n", + "from aer_worker import get_compiled_circuit, submit_single\n", "from tierkreis.builder import GraphBuilder\n", "\n", "\n", diff --git a/docs/source/examples/worker.ipynb b/docs/source/examples/worker.ipynb deleted file mode 100644 index ef4ebd6c8..000000000 --- a/docs/source/examples/worker.ipynb +++ /dev/null @@ -1,207 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "72c5f606", - "metadata": {}, - "source": [ - "# Writing tierkreis workers\n", - "One of the core concepts in tierkreis are workers.\n", - "In this example, we will cover the conceptual ideas of workers and how to write one.\n", - "For this we will write a simple worker from scratch.\n", - "The `tkr init` cli provides a convenient set up for this which will automate most of the tasks described after.\n", - "\n", - "## Concepts\n", - "\n", - "Workers encapsulate a set of functionalities which are described in their API (stubs).\n", - "The API contains a list of typed function interfaces which are implemented by the worker.\n", - "Tierkreis expects the functions to be stateless and will only checkpoint inputs and outputs of the function call.\n", - "Although, it is still possible to introduce state, but this then has to be managed by the programmer.\n", - "\n", - "Each worker is an independent program, can have multiple implementations (e.g. based on architecture) of these interfaces and has its own dependencies.\n", - "There are two main ways to write workers:\n", - "- Using the tierkreis `Worker` class for python based workers\n", - "- Using a spec file to define the interface for non-python workers\n", - "\n", - "In this example, we will write a simple worker, that greets a subject.\n", - "`greet: str -> str -> str `\n", - "\n", - "## Worker Setup\n", - "\n", - "A worker consist of four conceptual parts which we will set up in this example:=\n", - "1. Dependency information. We recommend using `uv` and a `pyproject.toml`.\n", - "2. A main entry point into the worker called `main.py`\n", - "3. The api definition of the worker typically called `stubs.py` or `api.py`. For python workers this can be generated from the main file.\n", - "4. (Optional) Library code containing the core logic.\n", - "\n", - "### 1.Dependencies\n", - "\n", - "Dependency information should be provide, such that the worker can run in its installed location.\n", - "For python, this is canonically done through the `pyproject.toml` which tierkreis also uses.\n", - "\n", - "```toml\n", - "[project]\n", - "name = \"hello-world-worker\"\n", - "version = \"0.1.0\"\n", - "requires-python = \">=3.12\"\n", - "dependencies = [\"tierkreis\"]\n", - "\n", - "[dependency-groups]\n", - "# Optional dev dependency group\n", - "dev = [\"ruff\"]\n", - "```\n", - "Ruff is an optional dependency to format the code later.\n", - "\n", - "### 2. Worker entry point\n", - "The entrypoint contains the implementations of the interface.\n", - "We are going to use the build in `Worker` class to define the worker.\n", - "Each worker needs a unique name, here we chose `hello_world_worker`.\n", - "In tierkreis the worker is defined by its name and the directory it lives in." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a4153a37", - "metadata": {}, - "outputs": [], - "source": [ - "%pip install tierkreis" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "48ab026a", - "metadata": {}, - "outputs": [], - "source": [ - "import logging\n", - "from pathlib import Path\n", - "\n", - "from tierkreis import Worker\n", - "\n", - "logger = logging.getLogger(__name__)\n", - "worker = Worker(\"hello_world_worker\")" - ] - }, - { - "cell_type": "markdown", - "id": "aa15cdbe", - "metadata": {}, - "source": [ - "From here we can add functions to the worker by decorating python functions with `@worker.task()`.\n", - "To make use of the type safety in tierkreis, you should provide narrow types to the function." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "deebc7b0", - "metadata": {}, - "outputs": [], - "source": [ - "@worker.task()\n", - "def greet(greeting: str, subject: str) -> str:\n", - " logger.info(\"%s %s\", greeting, subject)\n", - " return greeting + subject" - ] - }, - { - "cell_type": "markdown", - "id": "7167a5ee", - "metadata": {}, - "source": [ - "Now we have defined a greet function that concatenates a two strings.\n", - "It now lives in the namespace `hello_world_worker.greet`.\n", - "To make sure the worker runs correctly we need to add the following lines:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c3435525", - "metadata": {}, - "outputs": [], - "source": [ - "%%script false --no-raise-error\n", - "from sys import argv\n", - "if __name__ == \"__main__\":\n", - " worker.app(argv)\n" - ] - }, - { - "cell_type": "markdown", - "id": "d246abd5", - "metadata": {}, - "source": [ - "They make sure that tierkreis can call the worker correctly.\n", - "\n", - "### 3. Api definition\n", - "For python workers, tierkreis can automatically generate the api definition.\n", - "You can either run the app as `uv run main.py --stubs-path ./api/stubs.py` or call the function directly." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8de17d1e", - "metadata": {}, - "outputs": [], - "source": [ - "worker.namespace.write_stubs(Path(\"./api/stubs.py\"))" - ] - }, - { - "cell_type": "markdown", - "id": "1996bcbd", - "metadata": {}, - "source": [ - "This will generate a file `stubs.py` that contains the interface definition that can now be used as `task` in a tierkreis workflow.\n", - "The code will look like this:\n", - "```python\n", - "\"\"\"Code generated from hello_world_worker namespace. Please do not edit.\"\"\"\n", - "\n", - "from typing import NamedTuple\n", - "from tierkreis.controller.data.models import TKR\n", - "\n", - "\n", - "class greet(NamedTuple):\n", - " greeting: TKR[str] # noqa: F821 # fmt: skip\n", - " subject: TKR[str] # noqa: F821 # fmt: skip\n", - "\n", - " @staticmethod\n", - " def out() -> type[TKR[str]]: # noqa: F821 # fmt: skip\n", - " return TKR[str] # noqa: F821 # fmt: skip\n", - "\n", - " @property\n", - " def namespace(self) -> str:\n", - " return \"hello_world_worker\"\n", - "\n", - "```\n", - "Which defines a class describing the function." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "tierkreis (3.12.10)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.10" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/docs/source/executors/hpc.md b/docs/source/executors/hpc.md index 5a2f2faef..f1172acaa 100644 --- a/docs/source/executors/hpc.md +++ b/docs/source/executors/hpc.md @@ -81,7 +81,11 @@ JobSpec( ## Examples -In this section we give some examples based on commonly used job submission systems. -In cases where none of the following examples are useful, a custom executor can be provided to the Tierkreis library. +Tierkreis is compatibble with many schdeduling systems. +On the following pages you can find system-specific information. -[pjsub on Fugaku](./hpc/pjsub-fugaku.md) +```{toctree} +:maxdepth: 1 +:hidden: +./hpc/pjsub-fugaku.md +``` diff --git a/docs/source/executors/hpc/index.md b/docs/source/executors/hpc/index.md deleted file mode 100644 index 3593cde97..000000000 --- a/docs/source/executors/hpc/index.md +++ /dev/null @@ -1,7 +0,0 @@ -# HPC specific executors - -Documentatinon for HPC specific executors. - -```{toctree} -pjsub-fugaku.md -``` diff --git a/docs/source/executors/hpc/pjsub-fugaku.md b/docs/source/executors/hpc/pjsub-fugaku.md index d954983f0..7d1f1d13f 100644 --- a/docs/source/executors/hpc/pjsub-fugaku.md +++ b/docs/source/executors/hpc/pjsub-fugaku.md @@ -4,7 +4,7 @@ kernelspec: name: python3 --- -# Fugaku pjsub +# Fugaku: pjsub We see how to run a worker using `pjsub` on Fugaku. In particular we do not focus on the exact structure of our Tierkreis graph. diff --git a/docs/source/executors/index.md b/docs/source/executors/index.md index eaa765456..ee20ded7d 100644 --- a/docs/source/executors/index.md +++ b/docs/source/executors/index.md @@ -4,10 +4,10 @@ Executors are fundamental in running graph nodes in different environments. They also ensure that the workers can fulfill their contracts by preparing inputs and outputs. ```{toctree} -:maxdepth: 2 +:maxdepth: 1 +:hidden: shell.md hpc.md -hpc/index ``` ## The worker contract @@ -30,14 +30,14 @@ The controller is aware of the following files, if not specified otherwise they - The `definition` file contains the serialized `WorkerCallArgs`, the worker needs to parse this to find out about the locations inputs, outputs and the here listed files. - Completion is indicated by the `_done` file, workers must set this once they have written all outputs - Failures is indicated by the `_error` file, workers must set this if they can not complete normal execution -- In case of failure error messages should be written to the `errors_path` location of its call arguments, typically `///errors`. +- In case of failure error messages should be written to the `errors_path` location of its call arguments, typically `///logs`. Currently as a fallback it is also possible to write to the `///_errors` file. ### Task, Inputs and Outputs `WorkerCallArgs` contain the information of the function name of the task to call and it's inputs and the location to write outputs to. To supply workers with their inputs the `WorkerCallArgs` specify a mapping of input name to a location where the input is stored. -For example, the `greet` task of the [`hello_world_worker`](../worker/hello_world.md) expects two string inputs and outputs one file. +For example, the `greet` task of the [`hello_world_worker`](../examples/hello_world_graph.ipynb) expects two string inputs and outputs one file. The inputs are can be looked up by port name (`greeting`, `subject`) their values are stored in the output of other nodes in a a file `///outputs/`. The outputs of a worker follow the same pattern and are stored in the `output_dir` directory specified in the call args. For each value to output, there is an entry for it in the caller arguments output mapping. diff --git a/docs/source/executors/shell.md b/docs/source/executors/shell.md index fd4f9d95a..efc52cdac 100644 --- a/docs/source/executors/shell.md +++ b/docs/source/executors/shell.md @@ -36,7 +36,7 @@ Locations to inputs and outputs are provide in the form of Internally this uses the unix shell syntax for redirection ```sh -$> /path/to/binary +$> /path/to/binary ``` Beyond that, the executor does not provide further inputs, hence the script has to be able to handle the remainder itself. @@ -48,74 +48,5 @@ The rest will be discarded. ## Example -In this example we will use the shell executor to generate a RSA private key to sign a message. -For this we will use `openssl-genrsa`. -Additionally we will use the generated key to sign a message using the auth workers introduced in the [map example](../graphs/map.md). - -For the Shell Executor we need to wrap the `genrsa` in a `main.sh` script parsing the inputs and outputs: - -```sh -#!/usr/bin/env bash -numbits=$(cat $input_numbits_file) -openssl genrsa -out $output_private_key_file -aes128 -passout "file:$input_passphrase_file" $numbits -openssl rsa -in $output_private_key_file -passin "file:$input_passphrase_file" -pubout -out $output_public_key_file - -``` - -Then we can define the following graph. -Since we don't have stubs for the script (which we could generate by providing an [IDL] file) we use the untyped version using `Graphdata.func` - -```{code} ipython3 -from tierkreis.models import EmptyModel, TKR -from tierkreis.builder import GraphBuilder -from ..tutorial.auth_stubs import sign - -def signing_graph()-> GraphBuilder[EmptyModel, TKR[str]]: - g = GraphBuilder(EmptyModel, TKR[str]) - # Define the fix inputs to the graph - message = g.const("dummymessage") - passphrase = g.const(b"dummypassphrase") - - # Access the script, by calling genrsa inside the script openssl_workers - # We need to manually map the inputs to the correct variables - # These we have defined in the wrapper script - key_pair = g.data.func( - "openssl_worker.genrsa", - {"passphrase": passphrase.value_ref(), "numbits": g.const(4096).value_ref()}, - ) - # Similarly we parse the outputs as we have defined them in the script - private_key: TKR[bytes] = TKR(*key_pair("private_key")) # unsafe cast - public_key: TKR[bytes] = TKR(*key_pair("public_key")) # unsafe cast - - # Finally we use the auth worker to sign the message - signing_result = g.task(sign(private_key, passphrase, message)).hex_signature - g.outputs(signing_result) - return g -``` - -Running the graph follows all the usual steps. -We only want to use the shell executor to run the script, so we need to provide a default executor. - -```{code} ipython3 - -from pathlib import Path -from uuid import UUID -from tierkreis import run_graph -from tierkreis.executor import MultipleExecutor, UvExecutor, ShellExecutor -from tierkreis.storage import FileStorage, read_outputs - -graph = signing_graph() -storage = FileStorage(UUID(int=105), "sign_graph", do_cleanup=True) -# Make sure to provide the correct directory to the workers -registry_path = Path(__file__).parent / "example_workers" -uv = UvExecutor(registry_path, storage.logs_path) -shell = ShellExecutor(registry_path, storage.workflow_dir) -executor = MultipleExecutor(uv, {"shell": shell}, {"openssl_worker": "shell"}) -run_graph(storage, executor, graph().data, {}) - -# Verify the result -out = read_outputs(graph.get_data(), storage) -assert isinstance(out, str) -print(out) -assert len(out) == 1024 -``` +There is an example associated to this document. +You can find it [here](../examples/signing_graph.ipynb). diff --git a/docs/source/getting_started.md b/docs/source/getting_started.md deleted file mode 100644 index e524b0be3..000000000 --- a/docs/source/getting_started.md +++ /dev/null @@ -1,108 +0,0 @@ -# Getting Started - -## Project Setup - -Tierkreis works best with the [uv package manager](https://docs.astral.sh/uv/). We strongly recommend using it as your package manager for Tierkreis projects. - -To get started with Tierkreis start a new `uv` project in an empty directory with: - -```bash -uv init -``` - -Then add Tierkreis to the project and run the project setup tool. - -```bash -uv add tierkreis -uv run tkr init project -``` - -This will set up the following project structure for you: - -``` -project_root/ -├── tkr/ -│ ├── graphs/ -│ │ └── main.py -│ └── workers/ -│ └── example_worker/ -│ ├── api/ -│ │ ├── api.py (stubs for the worker) -│ │ ├── pyproject.toml -│ │ └── README.md -│ ├── tkr_example_worker_impl/ -│ │ ├── __init__.py -│ │ ├── impl.py (task definitions) -│ │ └── main.py -│ ├── __init__.py -│ ├── pyproject.toml -│ ├── README.md -│ └── uv.lock -├── .gitignore -├── .python-version -├── main.py -├── pyproject.toml -├── README.md -├── uv.lock -└── workflow_inputs.json -``` - -The repository is structure is intended to separate _graphs_, _workers_ and library code. - -From here you can run your first graph by running - -```bash -uv run tkr/graphs/main.py -> Value is: 1 -``` - -You can also run this through the cli: -```bash -uv run tkr run -o -> value: 1 -``` -This will use the following default locations: -1. The graph definition is used from `tkr/graphs/main.py:workflow` -2. The inputs are taken from `workflow_inputs.json` -3. `-o` enables the printing of outputs - - -From here you can continue with the other tutorials. - -## Tutorials for writing workflows - -A sequence of tutorials that cover the fundamentals of writing and operating Tierkreis workflows. - -[Our first graph](graphs/builtins.md) - -[Graph inputs and outputs](graphs/inputs.md) - -[Nested graphs using Eval](graphs/eval.md) - -[Iteration using Loop](graphs/loop.md) - -[Parallel computation using Map](graphs/map.md) - -## Tutorials for writing workers - -Tutorials on writing workers that provide additional tasks. -For a general overview look at the [worker documentation](worker/index.md) -For worker libraries see [this document](worker/native_workers/index.md) - -### Tierkreis Python library - -[Hello world worker](worker/hello_world.md) - -[Complex types in Tierkreis Python workers](worker/complex_types.md) - -### Other Workers - -[External workers with an IDL](worker/external_workers.md) - -## Executors - -[Overview](executors/index.md) - -[Shell Executors](executors/shell.md) - -[HPC Executors](executors/hpc.md) diff --git a/docs/source/graphs/builtins.md b/docs/source/graphs/builtins.md index 56c1e5947..3162e853d 100644 --- a/docs/source/graphs/builtins.md +++ b/docs/source/graphs/builtins.md @@ -4,7 +4,7 @@ kernelspec: name: python3 --- -# Graph using built-in tasks +# Tasks: Using Builtins To create this graph we need only to install the `tierkreis` package: @@ -62,7 +62,7 @@ g.outputs(three) ## Running the graph -To run a general Tierkreis graph we need to set up:- +To run a general Tierkreis graph we need to set up: - a way to store and share inputs and outputs (the 'storage' interface) - a way to run tasks (the 'executor' interface) diff --git a/docs/source/graphs/eval.md b/docs/source/graphs/eval.md index b78d83bff..18e1e9215 100644 --- a/docs/source/graphs/eval.md +++ b/docs/source/graphs/eval.md @@ -4,7 +4,7 @@ kernelspec: name: python3 --- -# Nested graphs using Eval +# Eval: Using nested graphs To create this graph we need only to install the `tierkreis` package: @@ -61,7 +61,7 @@ fib4.outputs(fourth.b) In the [next tutorial](./loop.md) we will see how to iterate programmatically. -# Execution +## Execution Since we still only use built-in functions, we execute the graph in the same way as before. diff --git a/docs/source/graphs/index.md b/docs/source/graphs/index.md index e33670e11..7d3341503 100644 --- a/docs/source/graphs/index.md +++ b/docs/source/graphs/index.md @@ -1,8 +1,10 @@ -# Beginners tutorial +# Graphs This tutorial will guide you through using the basic nodes for constructing a graph. ```{toctree} +:hidden: +:maxdepth: 1 builtins.md inputs.md eval.md diff --git a/docs/source/graphs/inputs.md b/docs/source/graphs/inputs.md index 8261193b4..ca4d04594 100644 --- a/docs/source/graphs/inputs.md +++ b/docs/source/graphs/inputs.md @@ -4,15 +4,7 @@ kernelspec: name: python3 --- -# Graph inputs and outputs - -To create this graph we need only to install the `tierkreis` package: - -``` -pip install tierkreis -``` - -# Graphs +# Inputs and Output ## Single input and single output @@ -144,7 +136,7 @@ g = GraphBuilder(TKR[str], MultiPortOutputData) g = GraphBuilder(MultiPortInputData, TKR[str]) ``` -# Execution +## Execution Since we still only use built-in functions, we execute the graph in the same way as before. For the examples with graph inputs, we provide the input in the third argument of `run_graph`. diff --git a/docs/source/graphs/loop.md b/docs/source/graphs/loop.md index ea2b95ac0..eea8f6567 100644 --- a/docs/source/graphs/loop.md +++ b/docs/source/graphs/loop.md @@ -4,7 +4,7 @@ kernelspec: name: python3 --- -# Iteration using Loop +# Loop: Iterating graphs One way to perform iteration in Tierkreis is to use `GraphBuilder.loop`. The first argument to `GraphBuilder.loop` is a graph that constitutes the loop body. @@ -124,7 +124,7 @@ loop_output = g.loop(body, LoopBodyInput(g.const(0))) g.outputs(loop_output.i) ``` -# Execution +## Execution Since we still only use built-in functions, we execute the graph in the same way as before. diff --git a/docs/source/graphs/map.md b/docs/source/graphs/map.md index f870845df..f022bb57d 100644 --- a/docs/source/graphs/map.md +++ b/docs/source/graphs/map.md @@ -4,7 +4,7 @@ kernelspec: name: python3 --- -# Parallel computation using Map +# Map: Parallel computation An example of how to run several CPU intensive nodes in parallel and aggregate the results. @@ -15,7 +15,7 @@ For demonstration purposes we ensure that the plaintexts are short enough (and t The following worker is in the [Tierkreis GitHub repo](https://github.com/Quantinuum/tierkreis) at `docs/source/examples/example_workers/auth_worker`: -```{literalinclude} ../examples/example_workers/auth_worker/src/main.py +```{literalinclude} ../examples/example_workers/auth_worker/tkr_auth_worker_impl/impl.py :language: python ``` @@ -25,7 +25,7 @@ Since this worker uses the Tierkreis Python library, we can automatically genera The stub files will provide us with type hints in the graph building process later on. ```{code-cell} -!cd ../examples/example_workers/auth_worker && uv run src/main.py --stubs-path ../../../example_workers/auth_worker/api/api.py > /dev/null 2>&1 +!cd ../examples/example_workers/auth_worker && uv run tkr_auth_worker_impl/main.py --stubs-path ./api/api.py > /dev/null 2>&1 ``` ## Writing a graph diff --git a/docs/source/index.md b/docs/source/index.md index 2201630a1..3ca2e2ce0 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -9,18 +9,33 @@ Tierkreis combines task-based workers with context dependent execution on variou ## Quick Start +To get up to speed with tierkreis we recommend the [Beginners Tutorial](./tutorial/index.md). +It will guide you through the core concepts of Tierkreis, how to construct graphs, define workers, and execute them. +For more advanced topics see the specific contents below. + ## Contents ```{toctree} :maxdepth: 3 installation.md -core_concepts.md -getting_started.md +tutorial/index +``` + +```{toctree} +:maxdepth: 2 +:caption: User guide +tutorial/core_concepts.md +tutorial/visualization.md +tutorial/logging_and_errors.md +tutorial/cli graphs/index -visualization.md -logging_and_errors.md worker/index executors/index -examples/index +tutorial/tutorials.md +``` + +```{toctree} +:maxdepth: 2 +:caption: API Reference apidocs/index ``` diff --git a/docs/source/installation.md b/docs/source/installation.md index fa9dd55bc..8dbac9804 100644 --- a/docs/source/installation.md +++ b/docs/source/installation.md @@ -4,11 +4,11 @@ Tierkreis consist of multiple packages: - `tierkreis`: the core functionality including the - `tierkreis-visualization`: the visualization library which is necessary for the debug view -- independent [workers](./core_concepts) consisting of API and implementation packages +- independent [workers](./tutorial/core_concepts) consisting of API and implementation packages - `tkr-*-worker`: **only the API** - `tkr-*-worker-impl`: API + Implementation -The libraries can be installed with `pip` but we recommend using [uv](https://docs.astral.sh/uv/) read more in the [Getting Started](./getting_started.md) docs. +The libraries can be installed with `pip` but we recommend using [uv](https://docs.astral.sh/uv/) read more in the [Getting Started](./tutorial/index.md) docs. ``` uv add tierkreis diff --git a/docs/source/tutorial/cli.md b/docs/source/tutorial/cli.md new file mode 100644 index 000000000..745d25521 --- /dev/null +++ b/docs/source/tutorial/cli.md @@ -0,0 +1,287 @@ +# TKR: Tierkreis CLI Tool + +Tierkreis includes a comprehensive command-line interface to manage workflows, workers, and other aspects of your project. +This document provides a complete reference for all available commands and options. + +## Overview + +The main `tkr` command has three primary subcommands: + +- **`init`** — Initialize and manage Tierkreis project resources +- **`run`** — Execute workflow graphs +- **`vis`** — Visualize and inspect graphs + +Use `--help` on any of the commands for a quick overview, or consult the sections below for detailed information about each command. + +--- + +## tkr init + +Initialize and manage Tierkreis project resources. The `init` command has three subcommands: `project`, `worker`, and `stubs`. + +### tkr init project + +Sets up a new Tierkreis project and manages project-wide options. Make sure to set up a Python project first using `uv init`. + +**Usage:** +```bash +uv run tkr init project [OPTIONS] +``` + +**Description:** +This command initializes the directory structure for a Tierkreis project, including directories for graphs and workers. You can customize the default locations for these resources. + +**Options:** + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `--project-directory` | PATH | `.` (current directory) | Root directory for the project | +| `--graphs-directory` | PATH | `./tkr/graphs` | Location where workflow graphs are stored | +| `--worker-directory` | PATH | `./tkr/workers` | Location where worker implementations are stored | +| `--default-checkpoint-directory` | PATH | `~/.tierkreis/checkpoints` | Where workflow checkpoints are saved. Also sets the `TKR_DIR` environment variable for the current shell | + +**Examples:** + +```bash +# Initialize with default directories +uv run tkr init project + +# Use custom directories +uv run tkr init project \ + --project-directory ./my-project \ + --graphs-directory ./my-graphs \ + --worker-directory ./my-workers + +# Set a custom checkpoint directory +uv run tkr init project \ + --default-checkpoint-directory /data/checkpoints + +# Persist checkpoint directory (add to shell config file, e.g., ~/.bashrc or ~/.zshrc) +export TKR_DIR=/data/checkpoints +``` + +### tkr init worker + +Generate a new worker template. +Workers are independent units that provide tasks to graphs. + +**Usage:** +```bash +uv run tkr init worker -n WORKER_NAME [OPTIONS] +``` + +**Description:** +Creates a new worker with boilerplate code. +By default, it generates a Python worker with a fixed project structure. +Use `--external` to generate an example typespec file instead for non-Python workers. + +**Options:** + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `-n, --worker-name` | STRING | — | Name of the worker to create | +| `--worker-directory` | PATH | `./tkr/workers` | Override the default worker directory | +| `--external` | FLAG | False | Generate an external (non-Python) worker with IDL file | + +**Examples:** + +```bash +# Create a Python worker +uv run tkr init worker -n my-worker + +# Create with custom directory +uv run tkr init worker -n alternative-worker --worker-directory ./custom_workers + +# Create an external (non-Python) worker +uv run tkr init worker -n external-worker --external +``` + +### tkr init stubs + +Generate the API files for workers using `uv`. +The APIs provide the task names during graph construction + +**Usage:** +```bash +uv run tkr init stubs [OPTIONS] +``` + +**Description:** +Scans the worker directory and generates python stub files (`api.py`) for type hints and IDE autocomplete. +This is **required** after creating or modifying worker interfaces. + +**Options:** + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `--worker-directory` | PATH | `./tkr/workers` | Directory to search for workers | +| `--api-file-name` | PATH | `./api/api.py` | Output location for generated API stubs (relative to worker directory) | + +```{warning} +Changing the api file name might break your worker. +Use only when appropriate. +``` + +**Examples:** + +```bash +# Generate stubs with defaults +uv run tkr init stubs + + +# Generate stubs for workers in custom directory +uv run tkr init stubs --worker-directory ./my-workers +``` +--- + +## tkr run + +Execute a workflow graph with specified inputs and configuration. + +**Usage:** +```bash +uv run tkr run [OPTIONS] +``` + +**Description:** +Runs a Tierkreis workflow graph. By default, it loads the graph from `./tkr/graphs/main.py:workflow` and reads inputs from `workflow_inputs.json`. + You can specify a different graph location or load from a JSON file, and provide inputs either as a JSON file or as binary files with key-value pairs. + +**Graph Specification:** + +You can specify the graph in two ways: + +1. **Module path (default):** `module.path:function_name` + - Example: `examples.hello_world:hello_graph` + - Example: `./my_graphs/workflow.py:my_workflow` + +2. **JSON file:** Use `-f/--from-file` to load a serialized graph + +**Input Specification:** + +You can provide inputs in two ways: + +1. **JSON file (default):** A single `.json` file containing input values + - Example: `workflow_inputs.json` + +2. **Binary files:** Key-value pairs where `key` is the input port name and `path` is a file containing binary data + - Example: `port1:input1 port2:input2` + +Where input1 might look like + +**Options:** + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `-g, --graph-location` | STRING | `./tkr/graphs/main.py:workflow` | Load graph from Python module or file | +| `-f, --from-file` | PATH | `workflow_inputs.json` | Load a serialized graph from a `.json` file | +| `-i, --input-files` | LIST | - | Input sources: `key:path` pairs | +| `-o, --print-output` | FLAG | False | Print the outputs of the top-level graph node | +| `--name` | STRING | — | Workflow name (for identification in logs) | +| `--run-id` | INT | — | Numeric run ID for tracking | +| `-n, --n-iterations` | INT | — | Maximum number of iterations for the workflow | +| `-p, --polling-interval-seconds` | FLOAT | — | Controller polling interval (tick rate) | +| `-r, --do-clean-restart` | FLAG | False | Clear all graph files before running | +| `--uv` | FLAG | False | Use `uv` as the executor | +| `--registry-path` | PATH | None | Location of executable tasks/binaries | +| `-l, --loglevel` | {CRITICAL, FATAL, ERROR, WARN, WARNING, INFO, DEBUG, NOTSET} | WARNING | Set logging level | +| `-v, --verbose` | FLAG | False | Enable verbose output | + +**Examples:** + +```bash +# Run with all defaults +uv run tkr run +# Loads from ./tkr/graphs/main.py:workflow and workflow_inputs.json + +# Print outputs to console +uv run tkr run -o +# Same as above but displays the workflow outputs + +# Run a specific graph +uv run tkr run -g my_module.workflows:main_workflow + +# Load graph from file +uv run tkr run -f my_graph.json + +# Provide custom inputs as binary files +uv run tkr run -i value:input.bin count:count.bin + +# Run with custom configuration +uv run tkr run \ + -g examples.quantum:circuit \ + -i inputs.json \ + --name "quantum-run-001" \ + -o -v + +# Clean restart and run with custom tick rate +uv run tkr run -r -p 0.5 --uv + +# Set logging level and run ID +uv run tkr run \ + -l DEBUG \ + --run-id 42 \ + --n-iterations 100 + +# Use custom registry path +uv run tkr run --registry-path /path/to/tasks +``` + +**What happens:** +1. The graph is loaded from the specified location +2. Inputs are loaded from the specified files +3. The workflow is executed by the Tierkreis controller +4. If `-o` is specified, outputs are printed to console +5. Checkpoints are saved to `TKR_DIR` (default: `~/.tierkreis/checkpoints`) + +--- + +## tkr vis + +Visualize and interact with Tierkreis graphs. + +**Usage:** +```bash +uv run tkr vis [OPTIONS] +``` + +**Description:** +Launches the Tierkreis visualizer, which provides an interactive web interface for viewing and debugging graphs. + This is useful for understanding workflow structure, inspecting node connections, and debugging execution flow. + +**Modes:** + +The visualization tool has t main modes: + +1. **Default Mode:** Start the visualization server showing all workflows run in the past. +w. **Graph Mode:** Visualize a specific graph object that has was not run. + +**Options:** + +| Option | Type | Description | +|--------|------|-------------| +| `--dev` | FLAG | Run in development mode with hot reload on code changes | +| `--graph` | STRING | Visualize a specific graph: `:` similar to `tkr run --g` | +| `-h, --help` | FLAG | Show help message | + +**Examples:** + +```bash +# Start the visualization server (interactive web UI) +uv run tkr vis + +# Start in development mode (with hot reload) +uv run tkr vis --dev + +# Visualize a specific graph +uv run tkr vis --graph ./my_graphs/workflow.py:main_workflow + +# Visualize a graph from a module +uv run tkr vis --graph my_module.graphs:visualization_graph +``` + +**Access the UI:** + +After running `uv run tkr vis`, the server starts at `http://localhost:8000` . +Open this URL in your web browser to access the visualization. +More info on the [visualizer](visualization.md) page. diff --git a/docs/source/core_concepts.md b/docs/source/tutorial/core_concepts.md similarity index 86% rename from docs/source/core_concepts.md rename to docs/source/tutorial/core_concepts.md index 1d7ce7ec6..e2daea7b4 100644 --- a/docs/source/core_concepts.md +++ b/docs/source/tutorial/core_concepts.md @@ -1,5 +1,8 @@ # Core Concepts in Tierkreis +This documents covers the basics of tierkreis. +If you want to immediately write you first graph, you can immediately skip to the [next page](../examples/first_graph.ipynb). + ## Program model In Tierkreis, a computation is represented as a sequence of tasks comprising a workflow. @@ -31,6 +34,10 @@ From the data dependencies, the runtime environment can infer which tasks it nee In Tierkreis, it is possible to associate types with edges (data) at construction. Type information can be used by the `GraphBuilder` to infer additional information about the graph to prevent runtime errors. +Tierkreis values correspond to the edges in the graph. +These values can have a type assigned to them at construction time. +The set of available types is a subset of all python types, e.g. we require serialization; see more in [complex types](../worker/complex_types.md) how to add your own serialization. +As result we use the `TKR` container to promote the python types into Tierkreis compatible types. ## Execution model @@ -63,7 +70,7 @@ The storage base class is defined in [](#tierkreis.controller.storage.protocol). A worker implements _atomic_ functionalities that will not be broken further by the controller. These functionalities are unrestricted and can be implemented in any language as long as they correctly implement the interface defined by the storage layer. -To facilitate the interface, workers have access to their own storage layer, see [](tierkreis.worker.storage.protocol). +To facilitate the interface, workers have access to their own storage layer, see the [API](#tierkreis.worker.storage.protocol). Typically, workers represent more expensive operations that run asynchronously. See [Workers](#tierkreis.worker.worker.Worker). Tierkreis can automatically generate type stubs for python workers. diff --git a/docs/source/tutorial/homeless.md b/docs/source/tutorial/homeless.md new file mode 100644 index 000000000..e1343a8ae --- /dev/null +++ b/docs/source/tutorial/homeless.md @@ -0,0 +1,23 @@ +### How to use the new project + +The project is set up in a way to be convenient for developers already experienced with Tierkreis. +Some of the concepts we will discuss in the [core concepts](./core_concepts.md) and implement in the upcoming pages. + +The repository is structure is intended to separate _graphs_, _workers_ and library code. +Graph definitions, like a `hello_world_graph` you will write in this tutorial should go into the `graphs` directory. +The main file contains an example graph, and has set up the storage and executors similar to the ones above. + +`Workers` are a way to add custom functionality, which will be executed as Tierkreis tasks. +If you're not familiar with `workers` yet, we will explain them as part of this tutorial. +It is very simple to wrap existing python code to make it available in Tierkreis. +Each worker is a separate entity, your new project will contain one `example_worker`. +If you want to include more they should have a similar structure. + + +## Advanced Topics + +Once you have finished the tutorial you can start writing your own workflows. +If you want to learn more details to fully leverage the power of Tierkreis, +the advanced user guide available [here](../tutorial_advanced/index.md). +It includes further tutorials, and detailed descsriptions on how to write graphs, workers and executors. + diff --git a/docs/source/tutorial/index.md b/docs/source/tutorial/index.md new file mode 100644 index 000000000..1c93525e7 --- /dev/null +++ b/docs/source/tutorial/index.md @@ -0,0 +1,105 @@ +# Beginners' Tutorial + +This tutorial will quickly show you the basics of using Tierkreis to +build and run a simple workflow. If you follow the whole sequence of +lessons you will have learned enough to get started on your own +projects straight away. + +Tierkreis can do many things that are not covered in this short +tutorial. For more in depth information, and more advanced topics, +please refer to the [User Guide](core_concepts.md) or the [API Reference](../apidocs/index). + +```{toctree} +:hidden: +:maxdepth: 1 +../examples/first_graph.ipynb +../examples/pytket_graph.ipynb +../examples/storage_and_executors.ipynb +../examples/hamiltonian.ipynb +whatnext.md +``` + +## Before you begin + +Tierkreis is based on Python, and we strongly recommend using the +[uv package manager](https://docs.astral.sh/uv/). +You can use Tierkreis without `uv`, it's just more complex and +difficult. For this tutorial we assume you have it available. + +Let's set up a new project and install Tierkreis. +```bash +uv init +uv add tierkreis +``` +The Tierkreis Visualizer is a separate package, so we'll install that too. +```bash +uv add tierkreis-visualization +``` + +The Tierkreis package includes CLI tool called `tkr`. The best way to +start a new Tierkreis development is using this tool to create a +suitable project directory. +```bash +uv run tkr init project +``` + +This will set up the basic project, described below. From here you +can run an example graph as test, using the CLI: +```bash +uv run tkr run -o +> value: 1 +``` + +### Setting up prepackaged workers + +Throughout the tutorial we're going to use some prepackade worker and define your own. +To make the examples run locally you need to install the workers first: +```bash +uv add tkr-aer-worker-impl tkr-pytket-worker-impl +``` + +**TODO: make sure these packages exist.** + +## The Default Project +The `tkr init project` command creates a directory structure like this: +``` +project_root/ +├── tkr/ +│ ├── graphs/ +│ │ └── main.py +│ └── workers/ +│ └── example_worker/ +│ ├── api/ +│ │ ├── api.py (stubs for the worker) +│ │ ├── pyproject.toml +│ │ └── README.md +│ ├── tkr_example_worker_impl/ +│ │ ├── __init__.py +│ │ ├── impl.py (task definitions) +│ │ └── main.py +│ ├── __init__.py +│ ├── pyproject.toml +│ ├── README.md +│ └── uv.lock +├── .gitignore +├── .python-version +├── main.py +├── pyproject.toml +├── README.md +├── uv.lock +└── workflow_inputs.json +``` + +This seems like a lot, but the default project contains a lot of +things we won't need in this tutorial. There are really two +directories of note here: +* `graphs` contains the graphs that define your workflow. Making + graphs is the main thing you will do when using Tierkreis. We'll + tackle this topic in the [next lesson](../examples/first_graph.ipynb). +* `workers` is where we add custom functionality to the graph. You + can think of a worker as a library that is available in your + workflow. Code for your own workers will live here, as we describe + in [lesson 2](../examples/pytket_graph.ipynb) and the [worker advanced guide](../worker/index.md). + +The summary above is fairly simplified; for a more information you may +want to look at [core concepts](./core_concepts.md). diff --git a/docs/source/logging_and_errors.md b/docs/source/tutorial/logging_and_errors.md similarity index 96% rename from docs/source/logging_and_errors.md rename to docs/source/tutorial/logging_and_errors.md index 877c347e5..34527094c 100644 --- a/docs/source/logging_and_errors.md +++ b/docs/source/tutorial/logging_and_errors.md @@ -61,7 +61,7 @@ In both cases, the controller will stop the execution, raising a `TierkreisError Error information is available in two places. When running [`run_graph`](#tierkreis.controller.run_graph) or [`run_workflow`](#tierkreis.cli.run_workflow.run_workflow), error information including a stack trace will be printed to `stdout`. -For example running the [Errors and Debugging](./examples/errors_and_debugging.ipynb) example will produce the following output +For example running the [Errors and Debugging](../examples/errors_and_debugging.ipynb) example will produce the following output ``` Graph finished with errors. @@ -123,4 +123,4 @@ If you're using the visualize to debug workflow, error information will be immed On the landing page, the workflows table will show you all nodes containing an error. In then workflow view a red node indicates an error. Logging information is available by double clicking a node; error information by pressing the "!" button on errored nodes. -For a guide to the visualize please refer to [this document](./visualization.md) +For a guide to the visualize please refer to [this document](../tutorial//visualization.md) diff --git a/docs/source/tutorial/tutorials.md b/docs/source/tutorial/tutorials.md new file mode 100644 index 000000000..f3cb6c764 --- /dev/null +++ b/docs/source/tutorial/tutorials.md @@ -0,0 +1,38 @@ +# Advanced Examples + +Once you have completed the beginners tutorial, you can look at advanced topics. + +```{toctree} +:hidden: +:maxdepth: 1 +../examples/types_and_defaults.ipynb +../examples/polling_and_dir.ipynb +../examples/errors_and_debugging.ipynb +../examples/restart.ipynb +../examples/signing_graph.ipynb +../examples/hpc.ipynb +../examples/scipy.ipynb +../examples/qsci.ipynb +``` + +## Building the examples + +In the given examples you will be developing code involving Tierkreis workers. +Whenever you see an import containing `*_worker` this means one of the workers will be invoked (except for the `builtin`s). +```{important} +To run the examples you will need to have the worker code available too. +The simplest way to set this up is to clone the entire repository before running any of the examples. +``` + +```bash +git clone https://github.com/Quantinuum/tierkreis.git +``` + +To set up the environment we use uv: + +```bash +cd tierkreis && uv sync --all-extras +``` + +Note that some third-party packages e.g. qulacs, automatically included via `uv sync --all-extras`, have other dependencies that you'll need to have installed on your system first: CMake and Boost (>=1.71). +When running the notebooks select the kernel corresponding to the uv environment. diff --git a/docs/source/visualization.md b/docs/source/tutorial/visualization.md similarity index 94% rename from docs/source/visualization.md rename to docs/source/tutorial/visualization.md index b8e101fbe..7933e659c 100644 --- a/docs/source/visualization.md +++ b/docs/source/tutorial/visualization.md @@ -61,7 +61,7 @@ Reloading is not available through the programming interface. ## Usage The visualizer provides plenty information about the graphs in the system. -![visualizer overview](./_static/visualization/overview.png) +![visualizer overview](../_static/visualization/overview.png) ### General Functionality @@ -87,7 +87,7 @@ Tasks will be displayed by their function names and constant values, inputs and The node status is indicated by the border color of the nodes: -![node state example](./_static/visualization/node_states.png) +![node state example](../_static/visualization/node_states.png) - Yellow: Node is currently running - Green: Node is finished @@ -110,7 +110,7 @@ The graph symbol indicates a value is a constant subgraph supplied as a nested g The higher order nodes `eval`, `map`, and `loop` can be expanded by pressing the `+` button. This will show their nested structure. -![node state example](./_static/visualization/Expanded.png) +![node state example](../_static/visualization/Expanded.png) For `eval` nodes this will immediately be the nested graph; For `map`/`loop` nodes this will show the individual elements/iterations which each contain their own subgraph. For unevaluated graphs, this will only show a placeholder evaluation. @@ -122,4 +122,4 @@ Logs can be accessed by double-clicking a node. If an error has occurred on a node, it will have a `!` button. Pressing it will show the error information. -![error logs](./_static/visualization/Debugging.png) +![error logs](../_static/visualization/Debugging.png) diff --git a/docs/source/tutorial/whatnext.md b/docs/source/tutorial/whatnext.md new file mode 100644 index 000000000..716311dc4 --- /dev/null +++ b/docs/source/tutorial/whatnext.md @@ -0,0 +1,54 @@ +# What Next? + +You've reached the end of the tutorial. You have learned how to +define a graph, fill it with tasks, and run the workflow. We only +scratched the surface of what you can do with Tierkreis, but you know +enough to begin using it for your own work. + +If you want to learn more details to fully leverage the power of +Tierkreis, the advanced user guide available +[here](user_guide.md). It includes further tutorials, +and detailed descriptions on how to write graphs, workers and +executors. + +You might find the follow topics useful next: + +* [Defining a worker from a shell command](../worker/index.md#generating-workers-from-the-cli) +* [Using HPC executors](../executors/hpc.md) +* [Working with the source distribution of Tierkreis](tutorials.md#building-the-examples) + + + +## Documents for writing graphs + +A sequence of documents that cover the fundamentals of writing complex Tierkreis graphs. + +[Using builtin functions](../graphs/builtins.md) + +[Defining graph inputs and outputs](../graphs/inputs.md) + +[Nested graphs using Eval](../graphs/eval.md) + +[Iteration using Loop](../graphs/loop.md) + +[Parallel computation using Map](../graphs/map.md) + +## Tutorials for writing workers + +Documents on writing workers that provide additional tasks. For a general overview look at the [worker documentation](../worker/index.md) + +[Complex types in Tierkreis Python workers](../worker/complex_types.md) + +[Tierkreis prebuild workers](../worker/native_workers/index.md) + +[External workers with an IDL](../worker/external_workers.md) + +## Executors + +Finally documentation on executors + +[A general overview](../executors/index.md) + +[Shell Executors](../executors/shell.md) + +[Running workers with HPC Executors](../executors/hpc.md) diff --git a/docs/source/worker/complex_types.md b/docs/source/worker/complex_types.md index 8644e7d68..526e184ae 100644 --- a/docs/source/worker/complex_types.md +++ b/docs/source/worker/complex_types.md @@ -152,7 +152,7 @@ Similarly to the `bytes` type, a top-level `ndarray` will produce a file contain If an `ndarray` is present within a nested Tierkreis structure then it will be serialized in the same way as `bytes` above (i.e. using the `__tkr_bytes__` discriminator). Unlike bytes, the stub generation process will produce `TKR[OpaqueType["numpy.ndarray"]]` for use in graph builder code. -# Custom serializers +## Custom serializers If the worker author would like to customize the (de)serialization functions they may use Python `Annotated` types. The Tierkreis Python library will look for subclasses of `tierkreis.controller.data.core.Serializer` and `tierkreis.controller.data.core.Deserializer` in the annotations. diff --git a/docs/source/worker/external_workers.md b/docs/source/worker/external_workers.md index 0f283977f..550345849 100644 --- a/docs/source/worker/external_workers.md +++ b/docs/source/worker/external_workers.md @@ -20,7 +20,7 @@ Tierkreis is restricted to a subset of the full specification: For example you could define the following interface in `TestNamespace/namespace.tsp` -```tsp +```rust @portmapping model Person { name: string; @@ -86,7 +86,7 @@ This is indicated with an `_error` file in the node location. This example shows how to parse the following interface: -```tsp +```rust interface MyWorker { double(value: Array): Array; diff --git a/docs/source/worker/hello_world.md b/docs/source/worker/hello_world.md deleted file mode 100644 index 57671b94e..000000000 --- a/docs/source/worker/hello_world.md +++ /dev/null @@ -1,108 +0,0 @@ ---- -file_format: mystnb -kernelspec: - name: python3 ---- - -# Hello world worker - -We see how to run tasks that are not Tierkreis built-in tasks. -Although we can build workers that manually read and write to the appropriate files, -in this section we use the helper functions in the Tierkreis Python library. - -## Worker code - -Our worker will consist of a single function returning a string. -We first instantiate a `Worker` class, which is constructed using the name of the worker. -The name of the worker tells the executor which worker a task comes from. -Therefore all of the different workers used in a single graph should have distinct names. - -```{code} ipython3 -from tierkreis import Worker - -worker = Worker("hello_world_worker") -``` - -The `Worker.task` method is used as a decorator to add functions to the worker. - -```{code} ipython3 -@worker.task() -def greet(greeting: str, subject: str) -> str: ... -``` - -```{note} -There are restrictions on the types of functions that `Worker.task` will accept. -If the function arguments and return types correspond to JSON types then the function will be accepted. -In addition there are a few ways of using more complex data structures, -but for now we keep things simple. -For more information please see [Complex types in Tierkreis Python workers](complex_types.md). -``` - -We use the `Worker.app` method to turn our Python programme into a legitimate Tierkreis worker. - -```{code} ipython3 -from sys import argv - -if __name__ == "__main__": - worker.app(argv) -``` - -The complete worker file is as follows: - -```{literalinclude} ../examples/example_workers/hello_world_worker/src/main.py -:language: python -``` - -### Logging and Errors - -The Tierkreis controller will automatically collect logs and errors from workers by adding a file handler, additionally redirecting any worker output from `stdout` and `stderr` to `errors_path` and `logs_path` automatically. -Any other output, e.g. writing debug information to a log file won't be captured. -For convenience you can use the builtin logging functionality `worker.logger` which is an instance of a `logging.Logger`. -Raising an exception in a worker task will terminate the graph execution. -An example of this can be found in the examples: `error_handling_graph.py` - -## Generating stubs - -Since this worker uses the Tierkreis Python library, we can automatically generate stub files using the following command. - -```{code-cell} -!cd ../examples/example_workers/hello_world_worker && uv run tkr_hello_world_worker_impl/main.py --stubs-path ../../../worker/hello_stubs.py > /dev/null 2>&1 -``` - -## Graph creation - -Now can we import the `greet` function from the stubs file and use it in `GraphBuilder.task`. - -```{code-cell} ipython3 -from tierkreis.builder import GraphBuilder -from tierkreis.models import TKR -from hello_stubs import greet - -g = GraphBuilder(TKR[str], TKR[str]) -output = g.task(greet(greeting=g.const("hello "), subject=g.inputs)) -g.outputs(output) -``` - -## Execution - -We use the `FileStorage` as usual but this time use the `UvExecutor` instead of the `ShellExecutor`. -To configure the `UvExecutor` we provide a path to a 'registry' folder of workers constructed using `uv`. -To add a worker to the registry we create a sub-folder of the registry folder containing a `main.py` that is executable by `uv`. -(The directory could be a whole `uv` project with a `pyproject.toml` or it might be that the `main.py` is a `uv` script.) -The folder name should correspond to the name of the worker. - -```{code-cell} -from uuid import UUID -from pathlib import Path - -from tierkreis import run_graph -from tierkreis.executor import UvExecutor -from tierkreis.storage import FileStorage, read_outputs - -storage = FileStorage(UUID(int=99), "hello_world_tutorial", do_cleanup=True) -executor = UvExecutor( - registry_path=Path("../examples/example_workers"), logs_path=storage.logs_path -) -run_graph(storage, executor, g.data, "world!") -read_outputs(g, storage) -``` diff --git a/docs/source/worker/index.md b/docs/source/worker/index.md index 65c79c10e..a16f1e7b3 100644 --- a/docs/source/worker/index.md +++ b/docs/source/worker/index.md @@ -1,3 +1,9 @@ +--- +file_format: mystnb +kernelspec: + name: python3 +--- + # Workers A worker implements _atomic_ functionalities that will not be broken further by the controller. @@ -13,12 +19,17 @@ As long as there is a runnable binary, you can provide a thin wrapper which allo ``` ```{toctree} -:maxdepth: 2 +:hidden: +:maxdepth: 1 complex_types.md external_workers.md -hello_world.md ../examples/multiple_outputs.ipynb -native_workers/index +native_workers/aer_worker.md +native_workers/ibmq_worker.md +native_workers/nexus_worker.md +native_workers/pytket_worker.md +native_workers/quantinuum_worker.md +native_workers/qulacs_worker.md ``` ## Generating workers from the cli @@ -28,7 +39,7 @@ By default, we assume workers are stored in a directory `/tkr/work You can generate a new worker by running: -``` +```bash tkr init worker --worker-name ``` @@ -65,13 +76,13 @@ When writing a workflow you don't need to call this function directly. Instead you need to provide the so-called function **stubs** to the task definition, which are available in `api.py` You can generate the stubs from the cli: -``` +```bash tkr init stubs ``` or running -``` +```bash uv run /tkr__impl/main.py --stubs-path .py ``` @@ -82,6 +93,12 @@ You can then import them using python: from worker_name import worker_function ``` +For example we can run the following to generate the api for the `hello_world_worker` + +```{code-cell} +!cd ../examples/example_workers/hello_world_worker && uv run tkr_hello_world_worker_impl/main.py --stubs-path ../../../worker/hello_stubs.py > /dev/null 2>&1 +``` + ### Using workers in multiple projects You need to write workers only once. @@ -93,6 +110,58 @@ When running you need to specify the correct registry for the executor or add th As alternative, you are free to publish the worker packages on pypi and add them as a prepackaged worker. +## Worker code + +Writing workers requires adding the `task` decorator. +Here we will provide a simple example, the full code is in [](../examples/hello_world_graph.ipynb) +Our worker will consist of a single function returning a string. +We first instantiate a `Worker` class, which is constructed using the name of the worker. +The name of the worker tells the executor which worker a task comes from. +Therefore all of the different workers used in a single graph should have distinct names. + +```{code} ipython3 +from tierkreis import Worker + +worker = Worker("hello_world_worker") +``` + +The `Worker.task` method is used as a decorator to add functions to the worker. + +```{code} ipython3 +@worker.task() +def greet(greeting: str, subject: str) -> str: ... +``` + +```{note} +There are restrictions on the types of functions that `Worker.task` will accept. +If the function arguments and return types correspond to JSON types then the function will be accepted. +In addition there are a few ways of using more complex data structures, +but for now we keep things simple. +For more information please see [Complex types in Tierkreis Python workers](complex_types.md). +``` + +### Logging and Errors + +The Tierkreis controller will automatically collect logs and errors from workers by adding a file handler, additionally redirecting any worker output from `stdout` and `stderr` to `errors_path` and `logs_path` automatically. +Any other output, e.g. writing debug information to a log file won't be captured. +For convenience you can use the builtin logging functionality `worker.logger` which is an instance of a `logging.Logger`. +Raising an exception in a worker task will terminate the graph execution. +An example of this can be found in the [advanced examples](../examples/errors_and_debugging.ipynb) + +## Using worker tasks + +We can import tasks (here the `greet`) function from the api file and use it in `GraphBuilder.task`. + +```{code-cell} ipython3 +from tierkreis.builder import GraphBuilder +from tierkreis.models import TKR +from hello_stubs import greet + +g = GraphBuilder(TKR[str], TKR[str]) +output = g.task(greet(greeting=g.const("hello "), subject=g.inputs)) +g.outputs(output) +``` + ## Running Workers In general, running workers is associated with an executor. @@ -106,6 +175,10 @@ If you used the cli to generate the worker layout described above, both cases ap This is due to worker being also added as a package to the root project. ``` +Configuring an executor requires sharing the storage interface (typically `FileStorage`) and setting the registry path. +This is typically in a shared location e.g., the `workers` directotry where each worker has its own folder. +The folder name must correspond to the name of the worker. + ### Running self defined workers For self defined python workers (using `main.py`) we use the `UvExecutor` as follows: @@ -199,7 +272,7 @@ will install an executable Python script `tkr_pytket_worker` into your virtual e **Example** -See the example `hamiltonian_graph.py`. +See the [example](../examples/hamiltonian.ipynb) #### Quantinuum Nexus diff --git a/docs/source/worker/native_workers/index.md b/docs/source/worker/native_workers/index.md deleted file mode 100644 index 93fbae250..000000000 --- a/docs/source/worker/native_workers/index.md +++ /dev/null @@ -1,16 +0,0 @@ -# Prepackaged workers - -These workers are natively available as stubs for use in Tierkreis Graphs. - -```{warning} -Currently only stubs are included. To use the workers during execution they need to be installed first. -``` - -```{toctree} -aer_worker.md -ibmq_worker.md -nexus_worker.md -pytket_worker.md -quantinuum_worker.md -qulacs_worker.md -``` diff --git a/justfile b/justfile index ae5b2bb22..bac0e047b 100644 --- a/justfile +++ b/justfile @@ -59,6 +59,7 @@ generate: just stubs-generate-api 'docs/source/examples/example_workers/error_worker/tkr_error_worker_impl' just stubs-generate-api 'docs/source/examples/example_workers/hello_world_worker/tkr_hello_world_worker_impl' just stubs-generate-api 'docs/source/examples/example_workers/multiple_outputs_worker/tkr_multiple_outputs_worker_impl' + just stubs-generate-api 'docs/source/examples/example_workers/pytket_example_worker/tkr_pytket_example_worker_impl' just stubs-generate-api 'docs/source/examples/example_workers/qsci_worker/tkr_qsci_worker_impl' just stubs-generate-api 'docs/source/examples/example_workers/scipy_worker/tkr_scipy_worker_impl' just stubs-generate-api 'docs/source/examples/example_workers/substitution_worker/tkr_substitution_worker_impl' diff --git a/uv.lock b/uv.lock index ba5728ff7..e78eff23b 100644 --- a/uv.lock +++ b/uv.lock @@ -26,6 +26,8 @@ members = [ "tkr-multiple-outputs-worker-impl", "tkr-nexus-worker", "tkr-nexus-worker-impl", + "tkr-pytket-example-worker", + "tkr-pytket-example-worker-impl", "tkr-pytket-worker", "tkr-pytket-worker-impl", "tkr-qsci-worker", @@ -3216,6 +3218,42 @@ requires-dist = [ { name = "tkr-nexus-worker", editable = "tierkreis_workers/nexus_worker/api" }, ] +[[package]] +name = "tkr-pytket-example-worker" +version = "0.2.0" +source = { editable = "docs/source/examples/example_workers/pytket_example_worker/api" } +dependencies = [ + { name = "tierkreis" }, +] + +[package.metadata] +requires-dist = [{ name = "tierkreis", editable = "tierkreis" }] + +[[package]] +name = "tkr-pytket-example-worker-impl" +version = "0.1.0" +source = { editable = "docs/source/examples/example_workers/pytket_example_worker" } +dependencies = [ + { name = "pydantic" }, + { name = "pytket" }, + { name = "pytket-qiskit" }, + { name = "ruff" }, + { name = "sympy" }, + { name = "tierkreis" }, + { name = "tkr-pytket-example-worker" }, +] + +[package.metadata] +requires-dist = [ + { name = "pydantic", specifier = ">=2.11.7" }, + { name = "pytket", specifier = ">=2.9.3" }, + { name = "pytket-qiskit" }, + { name = "ruff" }, + { name = "sympy" }, + { name = "tierkreis", editable = "tierkreis" }, + { name = "tkr-pytket-example-worker", editable = "docs/source/examples/example_workers/pytket_example_worker/api" }, +] + [[package]] name = "tkr-pytket-worker" version = "0.2.0"