Support running built-in or external "programs"

.github/workflows/generate_test_programs.yml

@@ -1,38 +1,66 @@
 name: Generate test programs
-env:
-  PYTHON_VERSION: "3.13"
-  POETRY_VERSION: "1.4"
+
 on:
   workflow_dispatch:
 
 jobs:
-  windows:
-    name: Build Windows Executables
-    runs-on: windows-latest
+  build:
+    name: Build on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-13, windows-latest]
+
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Set up python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-      - name: Cache dependencies
-        uses: hpcflow/github-support/init-cache@main
-        with:
-          name: make-test-programs
-          version: ${{ env.PYTHON_VERSION }}
-      - name: Set up poetry
-        uses: hpcflow/github-support/setup-poetry@main
+
+      - name: Set up MSVC (Windows)
+        if: runner.os == 'Windows'
+        uses: ilammy/msvc-dev-cmd@v1
+
+      - name: Download cJSON files
+        working-directory: hpcflow/data/programs/hello_world
+        run: |
+          mkdir cJSON
+          curl -sSL https://raw.githubusercontent.com/DaveGamble/cJSON/master/cJSON.c -o cJSON/cJSON.c
+          curl -sSL https://raw.githubusercontent.com/DaveGamble/cJSON/master/cJSON.h -o cJSON/cJSON.h
+        shell: bash
+
+      - name: Build program
+        if: runner.os != 'Linux'
+        working-directory: hpcflow/data/programs/hello_world
+        run: |
+          gcc hello_world.c cJSON/cJSON.c -I cJSON -o hello_world
+        shell: bash
+
+      - name: Build program (Rocky Linux 8)
+        if: runner.os == 'Linux'
+        uses: addnab/docker-run-action@v3
         with:
-          version: ${{ env.POETRY_VERSION }}
-      - name: Install dependencies
-        run: poetry install --without dev
-      - name: Build `hello_world.py` with pyinstaller for Windows
+          image: ghcr.io/hpcflow/rockylinux8-python:latest
+          options: -v ${{ github.workspace }}:/home --env GH_TOKEN=${{ secrets.GITHUB_TOKEN }}
+          run: |
+            cd /home/hpcflow/data/programs/hello_world
+            gcc hello_world.c cJSON/cJSON.c -I cJSON -o hello_world
+
+      - name: Take ownership of generated executable (linux - Rocky Linux 8)
         working-directory: hpcflow/data/programs/hello_world
-        run: poetry run pyinstaller --distpath ./dist/onefile --onefile --clean -y hello_world.py
-      - name: Upload executable artifact
-        id: upload-file
+        if: runner.os == 'Linux'
+        run: |
+          sudo chown $USER:$USER hello_world
+          chmod +r hello_world
+
+      - name: Test program
+        working-directory: hpcflow/data/programs/hello_world
+        run: |
+          echo '{ "p1": 10, "p2": 20, "p3": 30 }' > input.json
+          ./hello_world input.json output.json
+          cat output.json
+        shell: bash
+
+      - name: Upload `hello_world` artifact
         uses: actions/upload-artifact@v4
         with:
-          name: hello_world.exe
-          path: hpcflow/data/programs/hello_world/dist/onefile/hello_world.exe
+          name: hello_world_${{ runner.os }}${{ runner.os == 'Windows' && '.exe' || '' }}
+          path: hpcflow/data/programs/hello_world/hello_world${{ runner.os == 'Windows' && '.exe' || '' }}

hpcflow/app.py

@@ -34,6 +34,7 @@
     config_options=config_options,
     template_components=template_components,
     scripts_dir="data.scripts",  # relative to root package
+    programs_dir="data.programs",  # relative to root package
     workflows_dir="data.workflows",  # relative to root package
     demo_data_dir="hpcflow.data.demo_data",
     demo_data_manifest_dir="hpcflow.data.demo_data_manifest",

hpcflow/data/programs/hello_world/README.md

@@ -0,0 +1 @@
+This directory contains binary programs that are used to test the `Action.program` attribute. These binary programs are generated by the `generate_test_programs.yml` GitHub actions workflow (from `hello_world.c`), and are organised into sub-directories named by the `resources.platform` (e.g. "win", "linux", "macos") attribute.

hpcflow/data/programs/hello_world/hello_world.c

@@ -0,0 +1,87 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "cJSON.h"
+
+void hello_world()
+{
+    printf("hello, world\n");
+}
+
+void hello_world_ins_outs(const char *inputs_path, const char *outputs_path)
+{
+    printf("hello, world\n");
+
+    // Read input JSON
+    FILE *fp = fopen(inputs_path, "r");
+    if (!fp)
+    {
+        perror("Failed to open input file");
+        exit(1);
+    }
+
+    fseek(fp, 0, SEEK_END);
+    long len = ftell(fp);
+    fseek(fp, 0, SEEK_SET);
+
+    char *data = malloc(len + 1);
+    fread(data, 1, len, fp);
+    data[len] = '\0';
+    fclose(fp);
+
+    cJSON *json = cJSON_Parse(data);
+    if (!json)
+    {
+        fprintf(stderr, "Error parsing JSON input\n");
+        free(data);
+        exit(1);
+    }
+
+    double p1 = cJSON_GetObjectItem(json, "p1")->valuedouble;
+    double p2 = cJSON_GetObjectItem(json, "p2")->valuedouble;
+    double p3 = cJSON_GetObjectItem(json, "p3")->valuedouble;
+    double p4 = p1 + p2 + p3;
+
+    cJSON_Delete(json);
+    free(data);
+
+    // Create output JSON
+    cJSON *output_json = cJSON_CreateObject();
+    cJSON_AddNumberToObject(output_json, "p4", p4);
+
+    char *out_string = cJSON_Print(output_json);
+
+    fp = fopen(outputs_path, "w");
+    if (!fp)
+    {
+        perror("Failed to open output file");
+        cJSON_Delete(output_json);
+        free(out_string);
+        exit(1);
+    }
+
+    fprintf(fp, "%s\n", out_string);
+    fclose(fp);
+
+    cJSON_Delete(output_json);
+    free(out_string);
+}
+
+int main(int argc, char *argv[])
+{
+    if (argc == 1)
+    {
+        hello_world();
+    }
+    else if (argc == 3)
+    {
+        hello_world_ins_outs(argv[1], argv[2]);
+    }
+    else
+    {
+        fprintf(stderr, "Usage: %s [input.json output.json]\n", argv[0]);
+        return 1;
+    }
+
+    return 0;
+}

hpcflow/sdk/app.py

@@ -454,6 +454,8 @@ class BaseApp(metaclass=Singleton):
         Configuration options.
     scripts_dir:
         Directory for scripts.
+    programs_dir
+        Directory for programs.
     workflows_dir:
         Directory for workflows.
     demo_data_dir:
@@ -491,6 +493,7 @@ def __init__(
         gh_repo: str,
         config_options: ConfigOptions,
         scripts_dir: str,
+        programs_dir: str | None = None,
         workflows_dir: str | None = None,
         demo_data_dir: str | None = None,
         demo_data_manifest_dir: str | None = None,
@@ -522,6 +525,8 @@ def __init__(
         self.pytest_args = pytest_args
         #: Directory for scripts.
         self.scripts_dir = scripts_dir
+        #: Directory for programs.
+        self.programs_dir = programs_dir
         #: Directory for workflows.
         self.workflows_dir = workflows_dir
         #: Directory for demonstration data.
@@ -560,6 +565,7 @@ def __init__(
         self._environments: _EnvironmentsList | None = None
         self._task_schemas: _TaskSchemasList | None = None
         self._scripts: dict[str, Path] | None = None
+        self._programs: dict[str, Path] | None = None
 
         self.__app_type_cache: dict[str, type] = {}
         self.__app_func_cache: dict[str, Callable[..., Any]] = {}
@@ -1723,6 +1729,7 @@ def _load_template_components(self, *include: str) -> None:
                 "environments",
                 "task_schemas",
                 "scripts",
+                "programs",
             )
 
         self.logger.debug(f"Loading template components: {include!r}.")
@@ -1777,6 +1784,11 @@ def _load_template_components(self, *include: str) -> None:
             self._template_components["scripts"] = scripts
             self._scripts = scripts
 
+        if "programs" in include:
+            programs = self._load_programs()
+            self._template_components["programs"] = programs
+            self._programs = programs
+
         self.logger.info(f"Template components loaded ({include!r}).")
 
     @classmethod
@@ -1835,6 +1847,15 @@ def scripts(self) -> dict[str, Path]:
         assert self._scripts is not None
         return self._scripts
 
+    @property
+    def programs(self) -> dict[str, Path]:
+        """
+        The known programs.
+        """
+        self._ensure_template_component("programs")
+        assert self._programs is not None
+        return self._programs
+
     @property
     def task_schemas(self) -> _TaskSchemasList:
         """
@@ -2234,16 +2255,20 @@ def reload_config(
         self._load_config(config_dir, config_key, **overrides)
 
     @TimeIt.decorator
-    def _load_scripts(self) -> dict[str, Path]:
-        """
-        Discover where the built-in scripts all are.
-        """
+    def __load_builtin_files_from_nested_package(
+        self, directory: str | None
+    ) -> dict[str, Path]:
+        """Discover where the built-in files are (scripts or jinja templates)."""
         # TODO: load custom directories / custom functions (via decorator)
-        scripts_package = f"{self.package_name}.{self.scripts_dir}"
 
-        scripts: dict[str, Path] = {}
+        # must include an `__init__.py` file:
+        package = f"{self.package_name}.{directory}"
+
+        out: dict[str, Path] = {}
+        if not directory:
+            return out
         try:
-            with get_file_context(scripts_package) as path:
+            with get_file_context(package) as path:
                 for dirpath, _, filenames in os.walk(path):
                     dirpath_ = Path(dirpath)
                     if dirpath_.name == "__pycache__":
@@ -2252,11 +2277,25 @@ def _load_scripts(self) -> dict[str, Path]:
                         if filename == "__init__.py":
                             continue
                         val = dirpath_.joinpath(filename)
-                        scripts[val.relative_to(path).as_posix()] = Path(val)
+                        out[val.relative_to(path).as_posix()] = Path(val)
         except ModuleNotFoundError:
-            self.logger.exception("failed to find scripts package")
-        SDK_logger.info(f"loaded {len(scripts)} scripts from {scripts_package}")
-        return scripts
+            self.logger.exception(f"failed to find built-in files at {package}.")
+        SDK_logger.info(f"loaded {len(out)} files from {package}.")
+        return out
+
+    @TimeIt.decorator
+    def _load_scripts(self) -> dict[str, Path]:
+        """
+        Discover where the built-in scripts are.
+        """
+        return self.__load_builtin_files_from_nested_package(self.scripts_dir)
+
+    @TimeIt.decorator
+    def _load_programs(self) -> dict[str, Path]:
+        """
+        Discover where the built-in programs are.
+        """
+        return self.__load_builtin_files_from_nested_package(self.programs_dir)
 
     def _get_demo_workflows(self) -> dict[str, Path]:
         """Get all builtin demo workflow template file paths."""