feat: worker abstractions; fix: various improvements and bug fixes

.github/workflows/maintests.yml

@@ -19,23 +19,26 @@ jobs:
   build:
     env:
       AIWORKER_CACHE_HOME: ${{ github.workspace }}/.cache
-      TESTS_ONGOING: 1
-      HORDE_SDK_TESTING: 1
-      HORDE_MODEL_REFERENCE_MAKE_FOLDERS: 1
+      TESTS_ONGOING: "1"
+      AI_HORDE_TESTING: "1"
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python: ["3.10", "3.11", "3.12", "3.13"]
+        python-version:
+          - "3.12"
+          - "3.13"
 
     steps:
-      - uses: actions/checkout@v3
-      - name: Setup Python
-        uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+      - name: Install uv and set the python version
+        uses: astral-sh/setup-uv@v6
         with:
-          python-version: ${{ matrix.python }}
-      - name: Install tox and any other packages
-        run: |
-          python -m pip install --upgrade pip
-          pip install --upgrade -r requirements.dev.txt
-      - name: Run unit tests
-        run: tox -e tests-no-api-calls
+          python-version: ${{ matrix.python-version }}
+          enable-cache: true
+
+      - name: Install the project
+        run: uv sync --locked --all-extras --dev
+
+
+      - name: Run tests
+        run: uv run pytest tests --ignore-glob='**/*api_calls*' -m "not api_side_ci"

.github/workflows/prtests.yml

@@ -1,7 +1,7 @@
 name: Unstable Tests
 
 on:
-  pull_request_target:
+  pull_request:
     branches:
       - main
     types:
@@ -22,25 +22,27 @@ jobs:
   build:
     env:
       AIWORKER_CACHE_HOME: ${{ github.workspace }}/.cache
-      TESTS_ONGOING: 1
-      HORDE_SDK_TESTING: 1
-      HORDE_MODEL_REFERENCE_MAKE_FOLDERS: 1
+      TESTS_ONGOING: "1"
+      AI_HORDE_TESTING: "1"
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python: ["3.10", "3.11", "3.12", "3.13"]
+        python-version:
+          - "3.12"
+          - "3.13"
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-      - name: Setup Python
-        uses: actions/setup-python@v4
+      - name: Install uv and set the python version
+        uses: astral-sh/setup-uv@v6
         with:
-          python-version: ${{ matrix.python }}
-      - name: Install tox and any other packages
-        run: |
-          python -m pip install --upgrade pip
-          pip install --upgrade -r requirements.dev.txt
-      - name: Run unit tests
-        run: tox -e tests-no-api-calls
+          python-version: ${{ matrix.python-version }}
+          enable-cache: true
+
+      - name: Install the project
+        run: uv sync --locked --all-extras --dev
+
+      - name: Run tests
+        run: uv run pytest tests --ignore-glob='**/*api_calls*' -m "not api_side_ci"

.github/workflows/release.yml

@@ -39,12 +39,11 @@ jobs:
       run: echo "SETUPTOOLS_SCM_PRETEND_VERSION=${{ steps.release.outputs.version }}" >> $GITHUB_ENV
 
     # We need python...
-    - name: "🐍 Set up Python 3.10"
+    - name: "🐍 Set up Python 3.13"
       if: ${{ steps.release.outputs.version != '' }}
       uses: actions/setup-python@v4
       with:
-        python-version: "3.10"
-
+        python-version: "3.13"
     # Install build deps
     - name: "🛠 Install pypa/build"
       if: ${{ steps.release.outputs.version != '' }}

.gitignore

@@ -155,3 +155,9 @@ tests/testing_result_images/*
 workers.txt
 
 ai_horde_codegen.py
+
+.codiumai.toml
+
+.venv-*
+non_conforming_docstrings.json
+missing_original_docstrings.json

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "docs/haidra-assets"]
+	path = docs/haidra-assets
+	url = https://github.com/Haidra-Org/haidra-assets

.pre-commit-config.yaml

@@ -1,33 +1,37 @@
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
+    rev: v6.0.0
     hooks:
     -   id: check-yaml
+        exclude: ^mkdocs\.yml$
     -   id: end-of-file-fixer
     -   id: trailing-whitespace
--   repo: https://github.com/psf/black
-    rev: 24.10.0
-    hooks:
-    -   id: black
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.7.2
+    rev: v0.14.10
     hooks:
-    -    id: ruff
+    -    id: ruff-check
+    -    id: ruff-format
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v1.13.0'
+    rev: v1.19.1
     hooks:
     -   id: mypy
         args: [., --strict, --ignore-missing-imports, --exclude=^codegen]
         pass_filenames: false
         additional_dependencies: [
             pytest,
             pydantic>=2.9.2,
+            logfire,
             types-Pillow,
             types-requests,
             types-pytz,
             types-setuptools,
             types-urllib3,
             types-aiofiles,
+            types-PyYAML,
             StrEnum,
             horde_model_reference>=2.0.0,
         ]
+- repo: https://github.com/gauge-sh/tach-pre-commit
+  rev: v0.29.0
+  hooks:
+    - id: tach

.readthedocs.yaml

@@ -4,6 +4,10 @@
 # Required
 version: 2
 
+submodules:
+  include: all
+  recursive: true
+
 # Set the version of Python and other tools you might need
 build:
   os: ubuntu-22.04

CONTRIBUTING.md

@@ -1,56 +1,132 @@
 # Contributing to horde_sdk
 
+## Table of Contents
+
+- [Contributing to horde\_sdk](#contributing-to-horde_sdk)
+    - [Table of Contents](#table-of-contents)
+    - [Environment Management](#environment-management)
+        - [First time setup](#first-time-setup)
+    - [Code Quality Tools](#code-quality-tools)
+    - [Code Style and System Design](#code-style-and-system-design)
+    - [Testing](#testing)
+        - [When the API adds an endpoint or changes a model](#when-the-api-adds-an-endpoint-or-changes-a-model)
+        - [Verifying the horde SDK API surface](#verifying-the-horde-sdk-api-surface)
+    - [New Features/Pull Requests/Working on issues](#new-featurespull-requestsworking-on-issues)
+        - [Pull Request Do's and Don'ts](#pull-request-dos-and-donts)
+            - [Do](#do)
+            - [Don't](#dont)
+        - [Before Requesting Review](#before-requesting-review)
+
+## Environment Management
+
+[uv](https://github.com/astral-sh/uv/) is the suggested python environment management tool.
+
+### First time setup
+
+- Install uv, as described [here](https://github.com/astral-sh/uv/#installation).
+- `uv python install 3.10 3.11 3.12 3.13`
+- `uv self update`
+- `uv sync --all-groups`
+- The `.venv/` directory will now be created with all project, development and documentation dependencies installed.
+    - Be sure to point your IDE to the python binary appropriate for your OS in this directory.
+
 ## Code Quality Tools
 
-* [tox](https://tox.wiki/)
-  - Creates virtual environments for CI or local pytest runs.
-    - Note that the CI does not current execute calls to the production API by default.
-  - Run `tox list` or see `tox.ini` for more info
-* [pre-commit](https://pre-commit.com/)
-  - Creates virtual environments for formatting and linting tools
-  - Run `pre-commit run --all-files` or see `.pre-commit-config.yaml` for more info.
-* [black](https://github.com/psf/black)
-  - Whitespace formatter
-* [ruff](https://github.com/astral-sh/ruff)
-  - Linting rules from a wide variety of selectable rule sets
-  - See `pyproject.toml` for the rules used.
-  - See all rules (but not necessarily used in the project) availible in rust [here](https://beta.ruff.rs/docs/rules/).
-* [mypy](https://mypy-lang.org/)
-  - Static type safety
-  - I recommending using the [mypy daemon](https://mypy.readthedocs.io/en/stable/mypy_daemon.html) instead of periodically running `pre-commit` (or `mypy` directly.).
-    - If you are using VSCode, I recommend the `matangover.mypy` extension, which implements this nicely.
+- [**tox**](https://tox.wiki/)
+    - Creates virtual environments for CI or local pytest runs.
+        - Note that the CI does not current execute calls to the production API by default.
+    - Run `tox list` or see `tox.ini` for more info
+- [**pre-commit**](https://pre-commit.com/)
+    - Creates virtual environments for formatting and linting tools
+    - Run `pre-commit run --all-files` or see `.pre-commit-config.yaml` for more info.
 
-## Things to know
+> Note: Many of the tools below are run by `pre-commit` automatically, but can also be run manually if desired.
 
-  * The `AI_HORDE_DEV_URL` environment variable overrides `AI_HORDE_URL`. This is useful for testing changes locally.
-  * pytest files which end in `_api_calls.py` run last, and never run during the CI. It is currently incumbent on individual developers to confirm that these tests run successfully locally. In the future, part of the CI will be to spawn an AI-Horde and worker instances and test it there.
+- [**black**](https://github.com/psf/black)
+    - Whitespace formatter/code style formatter
+    - Run with `black .`
+- [**ruff**](https://github.com/astral-sh/ruff)
+    - Linting rules from a wide variety of selectable rule sets
+    - See `pyproject.toml` for the rules used.
+    - See all rules (but not necessarily used in the project) availible in rust [here](https://beta.ruff.rs/docs/rules/).
+    - Run with `ruff check .`
+        - Note: When using autofixing (`ruff check . --fix`), changes may be made that require running black, which can then result in needing to run `ruff check . --fix` again.
+        - Consider running `black . && ruff check . --fix && black . && ruff check . --fix` to avoid this.
+- [**mypy**](https://mypy-lang.org/)
+    - Static type safety
+    - I recommending using the [mypy daemon](https://mypy.readthedocs.io/en/stable/mypy_daemon.html) instead of periodically running `pre-commit` (or `mypy` directly.).
+- [**pyright**](https://github.com/microsoft/pyright)
+    - Shipped with vscode by default (via the python extension `ms-python.vscode-pylance`)
+    - Suggested settings:
+        - `"python.analysis.typeCheckingMode": "off"`
+            - The pylance extension has certain opinionated type checking assertions which are clash with mypy.
+            - For example, overriding an optional field to be non-optional is considered by pylance to be a type error due to the field being invariant and the parent class potentially settings it to `None`. However, by convention in the SDK, this is a forbidden pattern.
+        - `"python.analysis.languageServerMode": "full"`
+        - `"python.testing.pytestEnabled": true`
+- [**tach**](https://github.com/gauge-sh/tach)
+    - Enforces internal namespace dependency constraints. This helps avoid circular dependencies and helps ensure implementations are in a logical place.
 
+## Code Style and System Design
 
-## Verifying the horde SDK API surface
+See the [style guide in the docs folder](docs/concepts/style_guide.md) or go to the [same place in the horde_sdk documentation](https://horde-sdk.readthedocs.io/en/latest/) for more information on the code style requirements and design patterns used in the SDK.
 
-You can run the following:
+## Testing
 
-```bash
-pytest -m "object_verify"
-```
+- horde_sdk uses [pytest](https://docs.pytest.org/en/stable/) for testing.
+- The `AI_HORDE_DEV_URL` environment variable overrides `AI_HORDE_URL`. This is useful for testing changes locally.
+- pytest files which end in `_api_calls.py` run last, and never run during the CI. It is currently incumbent on individual developers to confirm that these tests run successfully locally. In the future, part of the CI will be to spawn an AI-Horde and worker instances and test it there.
+- **_Rationale_**: Local runs of the test suite benefit from a way to avoid running tests dependent on a live API.
 
-This will run the tests which validate the objects defined in the SDK are:
-- In the appropriate place
-- Match the live API (or if `AI_HORDE_DEV_URL` that version of the API)
-- That the models are exposed via `__init__.py`
-- And will run any other tests which ensure internal consistency.
-  - This generally does not include specific object validation beyond what can be automatically derived directly from the API docs or from general conventions from the SDK itself.
-  - If adding objects, you should add tests more specific to the expected functionality of that endpoint and the `object_verify` tests should only be treated as the bare-minimum.
+### When the API adds an endpoint or changes a model
 
-## When the API adds an endpoint or changes a model
 With the top level directory (the one that contains `pyproject.toml`) as your working directory:
+
 ```bash
 python horde_sdk/scripts/write_all_payload_examples_for_tests.py
 python horde_sdk/scripts/write_all_response_examples_for_tests.py
 python docs/build_docs.py
 ```
-This will update the data found in `tests/test_data/` from the default horde URL, or if any of the override environment variables are set, from there.
+
+This will update the data found in `tests/test_data/` from the default horde URL, or if any of the override environment variables are set, from there. This includes writing example payloads and responses extrapolated from the live APIs.
 
 Running `build_docs.py` will update any automatically generated mkdocs documentation stubs or resources (such as the API Model <-> SDK Model map).
 
 Be sure to run the test suite (without any `*_api_calls.py` tests) after. You may also may want to just start with `pytest -m "object_verify"` (see also the section on verifying the horde SDK API surface).
+
+### Verifying the horde SDK API surface
+
+You can run the following:
+
+```bash
+pytest -m "object_verify" -s
+```
+
+> Note: The `-s` flag is important as it allows you to see the output of the tests, which can be helpful for debugging. Often, the tests will also print out corrective actions to take if the tests fail.
+
+## New Features/Pull Requests/Working on issues
+
+The Horde ecosystem is a collaborative effort made possible through volunteer effort. We welcome all contributions and permission is not needed to work on issues or submit pull requests. However, if there is activity on an issue, you should consider reaching out to that person to coordinate efforts and avoid duplicate/conflicting work. Additionally, please considering commenting on an issue to let others know you are working on it.
+
+### Pull Request Do's and Don'ts
+
+#### Do
+
+- Be bold in your contribution
+- Ensure your code complies with the [code style and system design](#code-style-and-system-design) guidelines
+- Open draft pull requests when you want early feedback on your approach
+- Include clear descriptions of changes made and reasoning behind them
+- Include tests for any new features or bug fixes
+- Update documentation for new features or changes to existing functionality
+- Use descriptive commit messages consistent with the project commit history, especially for medium-to-large changesets.
+    - While it is possible we will squash commits before merging, it is still helpful to have descriptive commit messages for review and opens the possibility of rebasing instead.
+
+#### Don't
+
+- Make large sweeping changes unrelated to your primary goal
+- Include unnecessary changes like formatting or edits to unrelated files (open a separate PR for this)
+
+### Before Requesting Review
+
+- Ensure your PR is based on the latest `main` branch (rebase if out of date)
+- Verify all tests and code quality checks pass against the intended API versions
+- Double-check that all necessary documentation is updated, including docstrings and the relevant markdown files in the `docs` directory.

docs/.pages

@@ -3,8 +3,25 @@ nav:
     - getting_started.md
     - faq.md
     - examples.md
-    - api_to_sdk_map.md
+    - Definitions: haidra-assets/docs/definitions
+    - AI-Horde:
+        - API Model to SDK Class Map: ai-horde/api_to_sdk_map
+        - AI Horde Request/Job Lifecycle: haidra-assets/docs/workers
+        - Kudos Explanation: haidra-assets/docs/kudos
+        - Motivations: haidra-assets/docs/why
+    - Workers:
+        - Generations: worker/generations
+        - Worker Loop Diagram: haidra-assets/docs/worker_loop
+    - SDK Development:
+        - Developer's Getting Started: developers.md
+        - CONTRIBUTING.md (github link): 'https://github.com/Haidra-Org/horde-sdk/blob/main/CONTRIBUTING.md'
+        - Style Guide: concepts/style_guide.md
+        - Package Structure: concepts/package_structure.md
+        - Definitions: definitions.md
+        - SDK Client Design: concepts/client_design
+        - Backend Values Mappers: concepts/backend_values_mappers
     - horde_sdk
+
     - GitHub Repo: 'https://github.com/Haidra-Org/horde-sdk'
 
 order: desc