fix(freebsd): make the rc.d scripts start their services

.github/scripts/freebsd-rc-test.sh

@@ -0,0 +1,160 @@
+#!/bin/sh
+#
+# End-to-end check of the contrib/freebsd rc.d scripts on a real FreeBSD host: installs both
+# services the way contrib/freebsd/README.md describes, runs them as the unprivileged `smoke`
+# account under daemon(8), and drives start/status/restart/stop through service(8).
+#
+# It installs binaries and rc.d scripts, adds a user, edits /etc/rc.conf and starts services, so
+# run it only as root on a throwaway FreeBSD VM (CI's freebsd-test job does), from the repo root.
+set -eu
+
+[ "$(uname -s)" = FreeBSD ] || { echo "freebsd-rc-test: FreeBSD only" >&2; exit 2; }
+[ "$(id -u)" = 0 ] || { echo "freebsd-rc-test: run as root" >&2; exit 2; }
+
+fail() { echo "FAIL: $*" >&2; exit 1; }
+ok() { echo "ok: $*"; }
+
+# poll SECONDS CMD...: succeed as soon as CMD does; fail once SECONDS have passed.
+poll() {
+	_left=$1; shift
+	while ! "$@" >/dev/null 2>&1; do
+		_left=$((_left - 1))
+		[ "$_left" -gt 0 ] || return 1
+		sleep 1
+	done
+}
+
+dump() {
+	_rc=$?
+	if [ "$_rc" -ne 0 ]; then
+		echo "---- daemon/smoke processes"
+		# shellcheck disable=SC2009 # want the user/ppid columns pgrep cannot print
+		ps -axww -o user,pid,ppid,command | grep -E '[d]aemon|[s]moke' || true
+		echo "---- /var/log/messages (tail)"
+		tail -n 40 /var/log/messages || true
+	fi
+	exit "$_rc"
+}
+trap dump EXIT
+
+supervisor() { cat "/var/run/$1/$1.pid"; }   # daemon(8) writes its own pid there (-P)
+child() { pgrep -P "$(supervisor "$1")"; }    # the binary daemon(8) supervises
+user_of() { ps -o user= -p "$1" | tr -d ' '; }
+cmd_of() { ps -ww -o command= -p "$1"; }
+api_up() { fetch -qo /dev/null http://127.0.0.1:8087/api/targets; }
+new_child() { _k=$(child "$1") && [ "$_k" != "$2" ]; }
+
+# daemon(8) -T sends the child's output to syslog under the service name; checkable only if
+# syslogd is running (it is on a stock FreeBSD install).
+if pgrep -x syslogd >/dev/null; then
+	have_syslog=1
+else
+	have_syslog=0
+	echo "note: syslogd is not running; skipping the syslog-tag checks"
+fi
+logged() { [ "$have_syslog" = 0 ] || grep -q "$1" /var/log/messages; }
+
+# ---- install, as contrib/freebsd/README.md documents
+go build -o /usr/local/bin/smoked ./cmd/smoked
+go build -o /usr/local/bin/smoke-agent ./cmd/smoke-agent
+install -m 0555 contrib/freebsd/smoked contrib/freebsd/smoke_agent /usr/local/etc/rc.d/
+mkdir -p /usr/local/share/heliograph
+rm -rf /usr/local/share/heliograph/web
+cp -R web /usr/local/share/heliograph/web
+id smoke >/dev/null 2>&1 || pw useradd smoke -c Heliograph -d /nonexistent -s /usr/sbin/nologin -w no
+
+# ---- smoked: no DSN, so it serves the built-in demo targets from memory
+service smoked onestart
+poll 30 api_up || fail "smoked API never came up on 127.0.0.1:8087"
+poll 10 service smoked onestatus || fail "service smoked onestatus: not running"
+fetch -qo - http://127.0.0.1:8087/api/targets | grep -q 'Cloudflare TCP :443' ||
+	fail "/api/targets lacks the demo targets"
+sup=$(supervisor smoked)
+kid=$(child smoked)
+[ "$(user_of "$sup")" = smoke ] || fail "daemon(8) runs as $(user_of "$sup"), want smoke"
+[ "$(user_of "$kid")" = smoke ] || fail "smoked runs as $(user_of "$kid"), want smoke"
+cmd_of "$kid" | grep -q -- '-serve -addr 127.0.0.1:8087 -webdir /usr/local/share/heliograph/web -downsample' ||
+	fail "unexpected smoked argv: $(cmd_of "$kid")"
+poll 10 logged 'smoked\[[0-9]*\]: .*serving' || fail "smoked output not in syslog under tag smoked"
+ok "smoked onestart: API serves demo targets; daemon(8) and smoked both run as smoke"
+
+kill "$kid"
+poll 15 new_child smoked "$kid" || fail "daemon(8) did not restart the killed smoked"
+poll 30 api_up || fail "API down after daemon(8) restarted smoked"
+ok "smoked: daemon(8) restarted a killed child"
+
+# v2.2.0 documented smoked_flags for smoked's own flags; it must still reach smoked, not daemon(8).
+sysrc smoked_flags="-resolve-ips"
+out=$(service smoked onerestart 2>&1) || fail "onerestart with smoked_flags failed: $out"
+echo "$out" | grep -q 'smoked_flags is passed to daemon(8)' || fail "no smoked_flags warning: $out"
+poll 30 api_up || fail "API down after onerestart"
+[ "$(supervisor smoked)" != "$sup" ] || fail "onerestart kept the old supervisor pid $sup"
+kid=$(child smoked)
+cmd_of "$kid" | grep -q -- '-webdir /usr/local/share/heliograph/web -resolve-ips' ||
+	fail "smoked_flags not passed to smoked: $(cmd_of "$kid")"
+sysrc -x smoked_flags
+ok "smoked onerestart: new supervisor; legacy smoked_flags reaches smoked with a warning"
+
+# SMOKED_DSN travels through su -m in the environment, never argv. With a DSN at a closed port
+# smoked waits (up to 60s) for the database, which leaves time to read its environment.
+sysrc smoked_dsn="postgres://smoke:ci-secret@127.0.0.1:1/smoke?sslmode=disable"
+service smoked onerestart
+poll 15 child smoked || fail "no smoked child with a DSN set"
+kid=$(child smoked)
+procstat -e "$kid" | grep -q 'SMOKED_DSN=postgres://smoke:ci-secret@127.0.0.1:1/smoke' ||
+	fail "SMOKED_DSN missing from smoked's environment: $(procstat -e "$kid")"
+if pgrep -f ci-secret >/dev/null; then   # pgrep never matches itself
+	fail "DSN password visible in a process's argv: $(pgrep -lf ci-secret)"
+fi
+sysrc -x smoked_dsn
+ok "smoked: DSN reaches the child's environment and stays out of argv"
+
+service smoked onestop
+if service smoked onestatus; then fail "smoked still running after onestop"; fi
+if pgrep -x smoked >/dev/null; then fail "a smoked process survived onestop"; fi
+[ ! -e /var/run/smoked/smoked.pid ] || fail "daemon(8) left /var/run/smoked/smoked.pid behind"
+ok "smoked onestop: supervisor and child gone, pidfile removed"
+
+# ---- smoke_agent: no hub here, so point it at a closed port with a throwaway client cert. It logs
+# the failed assignment pulls and keeps running, which is all the service plumbing needs.
+etc=/usr/local/etc/heliograph
+spool=/var/db/smoke-agent/spool
+mkdir -p "$etc"
+openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj /CN=ci-vantage \
+	-keyout "$etc/client.key" -out "$etc/client.pem" 2>/dev/null
+chown smoke "$etc/client.key"
+chmod 0600 "$etc/client.key"
+printf 'hub: https://127.0.0.1:9\nvantage: ci\n' > "$etc/agent.yaml"
+chown smoke "$etc/agent.yaml"
+chmod 0640 "$etc/agent.yaml"
+sysrc smoke_agent_args="-client-cert $etc/client.pem -client-key $etc/client.key -ca-cert $etc/client.pem"
+
+service smoke_agent onestart
+poll 15 test -e "$spool/spool.lock" || fail "smoke-agent never opened its spool at $spool"
+poll 10 service smoke_agent onestatus || fail "service smoke_agent onestatus: not running"
+sup=$(supervisor smoke_agent)
+kid=$(child smoke_agent)
+[ "$(user_of "$sup")" = smoke ] || fail "daemon(8) runs as $(user_of "$sup"), want smoke"
+[ "$(user_of "$kid")" = smoke ] || fail "smoke-agent runs as $(user_of "$kid"), want smoke"
+[ "$(stat -f %Su "$spool/spool.lock")" = smoke ] || fail "spool.lock not written by smoke"
+cmd_of "$kid" | grep -q -- "-config $etc/agent.yaml -spool-dir=$spool -client-cert $etc/client.pem" ||
+	fail "unexpected smoke-agent argv: $(cmd_of "$kid")"
+poll 10 logged 'smoke_agent\[[0-9]*\]: .*smoke-agent starting' ||
+	fail "smoke-agent output not in syslog under tag smoke_agent"
+ok "smoke_agent onestart: daemon(8) and smoke-agent run as smoke; spool written"
+
+# smoke_agent_spool="" must reach smoke-agent as an explicit empty -spool-dir= (in-memory mode).
+sysrc smoke_agent_spool=""
+service smoke_agent onerestart
+poll 15 child smoke_agent || fail "no smoke-agent child after onerestart"
+kid=$(child smoke_agent)
+cmd_of "$kid" | grep -q -- "-spool-dir= -client-cert" ||
+	fail "smoke_agent_spool=\"\" not passed as -spool-dir=: $(cmd_of "$kid")"
+sysrc -x smoke_agent_spool
+ok "smoke_agent onerestart: smoke_agent_spool=\"\" passes an empty -spool-dir="
+
+service smoke_agent onestop
+if service smoke_agent onestatus; then fail "smoke_agent still running after onestop"; fi
+if pgrep -x smoke-agent >/dev/null; then fail "a smoke-agent process survived onestop"; fi
+[ ! -e /var/run/smoke_agent/smoke_agent.pid ] || fail "daemon(8) left the smoke_agent pidfile behind"
+ok "smoke_agent onestop: supervisor and child gone, pidfile removed"

.github/workflows/ci.yml

@@ -465,8 +465,9 @@ jobs:
             --ignore-unfixed --exit-code 1 heliograph-caddy:ci
 
   # Real-FreeBSD runtime coverage. Both binaries are pure Go and cross-compile for FreeBSD, but a
-  # cross-compile only proves it BUILDS. This boots an actual FreeBSD VM on the Linux runner and runs
-  # the full test suite (and syntax-checks the rc.d scripts) so FreeBSD stays a *tested* target. The
+  # cross-compile only proves it BUILDS. This boots an actual FreeBSD VM on the Linux runner, runs
+  # the full test suite, then installs the contrib/freebsd rc.d scripts and drives both services
+  # through service(8) as the unprivileged `smoke` account, so FreeBSD stays a *tested* target. The
   # VM runs as root, so the raw-socket Ping path is reachable there. Independent of the image pipeline
   # (the published image is Linux-only), so a FreeBSD VM hiccup never blocks an image release — add
   # freebsd-test as a required status check to gate merges on FreeBSD.
@@ -494,6 +495,9 @@ jobs:
             go build ./...
             # DB / rrdtool integration tests self-skip without SMOKE_TEST_DSN / rrdtool (not installed here).
             go test ./...
+            # The rc.d scripts for real: install both services, then start/status/restart/stop them
+            # via service(8) as `smoke`, probe smoked's API, and check users, argv, env and pidfiles.
+            sh .github/scripts/freebsd-rc-test.sh
 
   # Attach native FreeBSD binaries to each GitHub Release. The published container image is Linux-only,
   # so FreeBSD hosts run a native binary; this builds, packages, and uploads them. Tag-only (v*),

AGENTS.md

@@ -62,6 +62,11 @@ go build -o smoked ./cmd/smoked
 SMOKED_BIN=./smoked node web/layout.test.mjs
 ```
 
+The CI `freebsd-test` job also runs `go test ./...` inside a FreeBSD VM, then
+`.github/scripts/freebsd-rc-test.sh`, which installs the `contrib/freebsd` rc.d scripts and drives both
+services through `service(8)`. That script changes the host it runs on (installs binaries, adds a user,
+edits `/etc/rc.conf`), so run it only as root on a throwaway FreeBSD VM, from the repo root.
+
 ### Integration tests (real TimescaleDB)
 
 The `pgstore` / `vantage` / `api` / `configstore` / `cmd/smoked --history` tests are **skipped

CHANGELOG.md

@@ -38,6 +38,22 @@ All notable changes to **Heliograph** are recorded here. The format follows
   With no `--config` and no targets in the DB config, rows are still keyed by SmokePing path, which
   matches a YAML target that has no `id`, and the run says so. History already imported under
   SmokePing paths is not moved; re-run `--history` to store it under the target ids.
+- **The FreeBSD `rc.d` scripts now actually start their services.** The v2.2.0 scripts in
+  [`contrib/freebsd/`](contrib/freebsd/) built a `daemon(8)` command line that could never run, for
+  three independent reasons: the unquoted two-word title (`-t Heliograph hub`) split, so `daemon(8)`
+  tried to run `hub` / `vantage` as the program; rc.subr hands `${name}_flags` to `$command`, so
+  `smoked_flags` (default `-downsample`) went to `daemon(8)`, which rejected it; and rc.subr started
+  `daemon(8)` through `su` as `smoke`, which then could not create `/var/run/<name>.pid` (and was also
+  given `-u smoke`). The scripts now follow the FreeBSD ports convention: rc.subr runs `daemon(8)` as
+  `${name}_user` with no `-u`, the pidfile lives in a `/var/run/<name>/` directory that user owns,
+  and the process title and syslog tag are the service name, so output is logged as `smoked` /
+  `smoke_agent` rather than `daemon`. **Knob rename:** extra flags move from `smoked_flags` /
+  `smoke_agent_flags` to `smoked_args` / `smoke_agent_args`. A leftover `_flags` value is still used,
+  with a warning. Also fixed: `smoke_agent_spool=""` now disables the on-disk spool as documented
+  (the old default assignment replaced an empty value, so the agent always spooled), and
+  `smoked_args=""` now drops the default `-downsample`. The `freebsd-test` CI job now installs both
+  scripts in its FreeBSD VM and drives start/status/restart/stop through `service(8)`, checking the
+  service user, argv, environment and pidfile, plus smoked's API.
 
 ### Security
 - **Revoking and re-adding a vantage now retires its earlier certificates.** Agent authorization

README.md

@@ -168,7 +168,9 @@ Postgres/TimescaleDB (`-dsn …`).
 Non-ICMP probes (`DNS`, `HTTP`, `TCPConnect`, `NTP`, …) need no special privilege.
 
 **Run it as a service.** [`contrib/freebsd/`](contrib/freebsd/) ships `rc.d` scripts and install
-notes for running `smoke-agent` (and `smoked`) under FreeBSD's service manager.
+notes for running `smoke-agent` (and `smoked`) under FreeBSD's service manager, supervised by
+`daemon(8)` as an unprivileged `smoke` account (set `smoke_agent_user=root` or `smoked_user=root`
+for the native `Ping` probe). Extra flags go in `smoke_agent_args` / `smoked_args`.
 
 ### Docker Compose
 

contrib/freebsd/README.md

@@ -3,7 +3,7 @@
 Both binaries are pure Go and run natively on FreeBSD (`amd64`/`arm64`). This directory
 ships `rc.d` service scripts so `smoke-agent` (a vantage) and `smoked` (the hub) run under
 FreeBSD's service manager, supervised by [`daemon(8)`](https://man.freebsd.org/cgi/man.cgi?daemon)
-(auto-restart on crash, output to syslog).
+(auto-restart on crash, output to syslog) as an unprivileged service account.
 
 | Script | Runs | Use when |
 | --- | --- | --- |
@@ -24,12 +24,31 @@ install -m 0755 smoked smoke-agent /usr/local/bin/
 pw useradd smoke -c "Heliograph" -d /nonexistent -s /usr/sbin/nologin -w no   # service account
 ```
 
+## How the scripts run the services
+
+rc.subr starts each service as `${name}_user` (default `smoke`) through `su(1)`, the same way
+ports such as `sysutils/node_exporter` do. Both `daemon(8)` and the binary it supervises therefore
+run as that account; nothing stays running as root. Consequences:
+
+- Everything the service reads (`agent.yaml`, a `smoked_config` file, the web directory) must be
+  readable by that account.
+- The pidfile is `/var/run/<name>/<name>.pid`, in a directory the service account owns; the script
+  creates it on start. It holds the `daemon(8)` supervisor's pid, so `service <name> stop` stops the
+  supervisor, which stops the binary without restarting it. `ps` shows the supervisor as
+  `daemon: <name>[<child pid>]`.
+- Output goes to syslog tagged with the service name (`smoked` / `smoke_agent`), facility `daemon`;
+  on a stock install it lands in `/var/log/messages`.
+- Extra command-line flags go in `smoked_args` / `smoke_agent_args`. Do **not** use
+  `smoked_flags` / `smoke_agent_flags`: rc.subr passes `${name}_flags` to `daemon(8)`, not to the
+  binary. (v2.2.0 documented the `_flags` names. A value left there is still used as the `_args`
+  value, with a warning, until you rename it.)
+
 ICMP: the native `Ping` probe needs a raw socket (run as root with `mode: privileged`); the
 `FPing` probe works unprivileged — `pkg install fping` installs a setuid-root `fping(8)`. Other
 probes (`DNS`, `HTTP`, `TCPConnect`, `NTP`, …) need no special privilege. **To use the native
 `Ping` probe under these services, also set `smoke_agent_user="root"` (or `smoked_user="root"`)** —
-the scripts default to the unprivileged `smoke` account and pass `-u smoke` to `daemon(8)`, and
-`mode: privileged` alone does not grant raw-socket access. The `FPing` probe needs no such override.
+`mode: privileged` alone does not grant raw-socket access to the default `smoke` account. The
+`FPing` probe needs no such override.
 
 ## Vantage agent (`smoke_agent`)
 
@@ -50,11 +69,13 @@ the scripts default to the unprivileged `smoke` account and pass `-u smoke` to `
    sysrc smoke_agent_enable="YES"
    sysrc smoke_agent_config="/usr/local/etc/heliograph/agent.yaml"
    service smoke_agent start
+   service smoke_agent status
    ```
 
 Knobs (set with `sysrc`): `smoke_agent_bin`, `smoke_agent_user` (default `smoke`),
-`smoke_agent_spool` (default `/var/db/smoke-agent/spool`; `""` disables the on-disk spool),
-`smoke_agent_flags` (extra flags). Logs go to syslog (`daemon` facility).
+`smoke_agent_spool` (default `/var/db/smoke-agent/spool`, created on start and owned by the service
+account; `""` disables the on-disk spool, even if `agent.yaml` sets `spool_dir`),
+`smoke_agent_args` (extra `smoke-agent` flags).
 
 ## Hub (`smoked`)
 
@@ -69,15 +90,18 @@ install -m 0555 smoked /usr/local/etc/rc.d/smoked
 sysrc smoked_enable="YES"
 sysrc smoked_dsn="postgres://smoke:smoke@db.example:5432/smoke?sslmode=disable"
 service smoked start
+service smoked status
 ```
 
 The dashboard/API is **unauthenticated**, so `smoked_addr` defaults to loopback
 (`127.0.0.1:8087`). To reach it beyond localhost, set `smoked_addr` (e.g. `sysrc
 smoked_addr=":8087"`) and front it with a TLS + Basic-Auth reverse proxy — never expose it
 directly. The DSN is passed via the environment (`SMOKED_DSN`), so the password stays out of `ps`;
 if the database is on another host, use an encrypted DSN (`sslmode=require` or `verify-full`, with
-the appropriate root/verify cert) rather than the `sslmode=disable` shown above.
+the appropriate root/verify cert) rather than the `sslmode=disable` shown above. Without
+`smoked_dsn`, `smoked` keeps the built-in demo targets in memory.
 
-Knobs: `smoked_bin`, `smoked_user`, `smoked_addr`, `smoked_webdir`
-(default `/usr/local/share/heliograph/web`), `smoked_config`, `smoked_flags`
-(default `-downsample`; add `-agent-addr :8443 -agent-hostname <host>` to accept remote vantages).
+Knobs: `smoked_bin`, `smoked_user` (default `smoke`), `smoked_addr`, `smoked_webdir`
+(default `/usr/local/share/heliograph/web`), `smoked_config`, `smoked_args`
+(default `-downsample`; `""` for none; add `-agent-addr :8443 -agent-hostname <host>` to accept
+remote vantages).