needlectl: add --profile and --config-dir options, support prod/dev multi-file compose stacks, default profile prod

merfanian · merfanian · commit 1bf941b102aa · 2025-05-30T10:48:05.000-07:00
- Introduce Profile enum and --profile/-P CLI option (default prod)
- Add --config-dir/-C flag to override config directory; defaults to NEEDLE_HOME/configs or profile-specific dev configs
- main() callback sets NEEDLE_CONFIG_DIR and NEEDLE_COMPOSE_FILES env vars for profile-based compose stacks
- DockerComposeManager now reads NEEDLE_COMPOSE_FILES and supports multiple -f args
- Remove deprecated add_volume method
- Update get_config_file to respect NEEDLE_CONFIG_DIR fallback
- Update README to document new flags and default profile behavior
diff --git a/needlectl/README.md b/needlectl/README.md
@@ -7,3 +7,39 @@ Needlectl is a command-line tool designed to interact with **Needle**, a system
 
 ---
 
+## ⚙️ Profiles & Configuration
+
+You can use the `--home`/`-H` flag to point needlectl at a custom Needle installation or local checkout, and select a runtime profile (`prod` or `dev`) that auto-configures the compose files and config directory. For advanced scenarios you can override just the configs path with `--config-dir`.
+
+By default, `--profile` is set to **prod**, so omitting it will assume the production profile.
+
+### Examples
+
+#### Development
+
+Spin up services against your local checkout in dev mode (fast configs + hot‑reload compose overrides):
+
+```bash
+needlectl --home $(pwd) --profile dev service start
+```
+
+This uses:
+- `NEEDLE_CONFIG_DIR=$NEEDLE_HOME/configs/fast`
+- Compose files: `docker/docker-compose.cpu.yaml` + `docker/docker-compose.dev.yaml`
+
+#### Production
+
+Run against a deployed install (standard configs + prod override):
+
+```bash
+needlectl --home /opt/needle --profile prod service start
+```
+
+#### Custom config-dir
+
+Only override the configuration directory (compose files follow the selected profile):
+
+```bash
+needlectl --home /opt/needle --config-dir /etc/needle/configs service start
+```
+
diff --git a/needlectl/cli/main.py b/needlectl/cli/main.py
@@ -1,7 +1,12 @@
 from typing import Optional
+from enum import Enum
+import os
+from pathlib import Path
 
 import typer
 
+from utils import get_storage_dir
+
 from cli.directory import directory_app
 from cli.generator import generator_app
 from cli.query import query_app
@@ -34,21 +39,67 @@ def version_callback(value: bool):
         raise typer.Exit()
 
 
+class Profile(str, Enum):
+    """Named runtime profiles for choosing prod vs dev compose/config setup."""
+    dev = "dev"
+    prod = "prod"
+
+
 @app.callback()
 def main(
-        ctx: typer.Context,
-        api_url: str = typer.Option("http://127.0.0.1:8000", help="API URL of the backend service."),
-        output: str = typer.Option("human", help="Output format: human|json|yaml"),
-        version: Optional[bool] = typer.Option(
-            None,
-            "--version",
-            "-v",
-            help="Show the backend version and exit.",
-            callback=version_callback,
-            is_eager=True,
-        ),
+    ctx: typer.Context,
+    home: Optional[str] = typer.Option(
+        None, "--home", "-H", help="Path to the Needle home directory"
+    ),
+    profile: Profile = typer.Option(
+        Profile.prod,
+        "--profile",
+        "-P",
+        help="Runtime profile (prod or dev). Default: prod",
+        case_sensitive=False,
+    ),
+    config_dir: Optional[str] = typer.Option(
+        None,
+        "--config-dir",
+        "-C",
+        help="Override configuration directory (defaults to $NEEDLE_HOME/configs or profile-specific)",
+    ),
+    api_url: str = typer.Option(
+        "http://127.0.0.1:8000", help="API URL of the backend service."
+    ),
+    output: str = typer.Option(
+        "human", help="Output format: human|json|yaml"
+    ),
+    version: Optional[bool] = typer.Option(
+        None,
+        "--version",
+        "-v",
+        help="Show the backend version and exit.",
+        callback=version_callback,
+        is_eager=True,
+    ),
 ):
+    if home:
+        os.environ["NEEDLE_HOME"] = home
+
+    needle_home = get_storage_dir()
+
+    if config_dir:
+        os.environ["NEEDLE_CONFIG_DIR"] = config_dir
+    elif profile == Profile.dev:
+        os.environ["NEEDLE_CONFIG_DIR"] = str(Path(needle_home) / "configs" / "dev")
+
+    files = []
+    if profile == Profile.dev:
+        files = [Path(needle_home) / "docker" / "docker-compose.cpu.yaml",
+                 Path(needle_home) / "docker" / "docker-compose.dev.yaml"]
+    else:
+        files = [Path(needle_home) / "docker" / "docker-compose.cpu.yaml",
+                 Path(needle_home) / "docker" / "docker-compose.prod.yaml"]
+    os.environ["NEEDLE_COMPOSE_FILES"] = os.pathsep.join(str(p) for p in files)
+
     ctx.obj = {
         "api_url": api_url,
-        "output": output.lower()
+        "output": output.lower(),
+        "profile": profile,
     }
diff --git a/needlectl/docker/docker_compose_manager.py b/needlectl/docker/docker_compose_manager.py
@@ -3,22 +3,32 @@
 import json
 import typer
 import yaml
+from pathlib import Path
 from utils import get_compose_file
 
 
 class DockerComposeManager:
     def __init__(self):
-        self.docker_compose_path = get_compose_file()
-        if not os.path.isfile(self.docker_compose_path):
-            typer.echo("Error: docker-compose file not found.")
-            raise typer.Exit(code=1)
+        compose_env = os.getenv("NEEDLE_COMPOSE_FILES")
+        if compose_env:
+            paths = compose_env.split(os.pathsep)
+            self.compose_files = [Path(p) for p in paths]
+        else:
+            self.compose_files = [get_compose_file()]
+
+        for path in self.compose_files:
+            if not path.is_file():
+                typer.echo(f"Error: docker-compose file not found: {path}")
+                raise typer.Exit(code=1)
 
-        # Load compose file to get service information
-        with open(self.docker_compose_path, "r") as file:
+        with open(self.compose_files[0], "r") as file:
             self.compose_data = yaml.safe_load(file)
 
     def _docker_compose_run(self, *args):
-        cmd = ["docker", "compose", "-f", self.docker_compose_path] + list(args)
+        cmd = ["docker", "compose"]
+        for path in self.compose_files:
+            cmd += ["-f", str(path)]
+        cmd += list(args)
         subprocess.run(cmd, check=True)
 
     def get_backend_version(self) -> str:
@@ -39,7 +49,10 @@ def get_backend_version(self) -> str:
                 return "unknown (backend service not found)"
 
             # Get container ID of the running backend service
-            cmd = ["docker", "compose", "-f", self.docker_compose_path, "ps", "-q", backend_service]
+            cmd = ["docker", "compose"]
+            for path in self.compose_files:
+                cmd += ["-f", str(path)]
+            cmd += ["ps", "-q", backend_service]
             container_id = subprocess.check_output(cmd).decode().strip()
 
             if not container_id:
@@ -78,28 +91,3 @@ def restart_containers(self):
     def log_services(self, service_name):
         self._docker_compose_run("logs", service_name)
 
-    def add_volume(self, service_name, volume_path):
-        """
-        Adds a volume to a specific service in the docker-compose file.
-        Args:
-            service_name (str): The name of the service to update.
-            volume_path (str): The path of the volume to add.
-        Raises:
-            typer.Exit: If the service is not found in the docker-compose file.
-        """
-        services = self.compose_data.get("services", {})
-        if service_name not in services:
-            typer.echo(f"Error: Service '{service_name}' not found in docker-compose.yml.")
-            raise typer.Exit(code=1)
-
-        service = services[service_name]
-        volumes = service.get("volumes", [])
-        # Check if the volume already exists
-        if volume_path not in [v.split(":")[0] for v in volumes if isinstance(v, str)]:
-            volumes.append(f"{volume_path}:{volume_path}")
-            service["volumes"] = volumes
-            with open(self.docker_compose_path, "w") as file:
-                yaml.safe_dump(self.compose_data, file, sort_keys=False)
-            typer.echo(f"Added volume '{volume_path}' to service '{service_name}'.")
-        else:
-            typer.echo(f"Volume '{volume_path}' already exists for service '{service_name}'.")
diff --git a/needlectl/utils.py b/needlectl/utils.py
@@ -13,11 +13,16 @@ def get_storage_dir():
 
 
 def get_config_file(filename) -> Path:
-    configs_path = Path(os.path.join(get_storage_dir(), "configs"))
-    if not os.path.exists(configs_path):
+    """Allow override of config directory via env var NEEDLE_CONFIG_DIR."""
+    config_base = os.getenv("NEEDLE_CONFIG_DIR")
+    if config_base:
+        configs_path = Path(config_base)
+    else:
+        configs_path = Path(os.path.join(get_storage_dir(), "configs"))
+    if not configs_path.exists():
         os.makedirs(configs_path, exist_ok=True)
 
-    return Path(os.path.join(configs_path, filename))
+    return configs_path / filename
 
 
 def get_compose_file():
