diff --git a/docs/PLAN_4c_renderers.md b/docs/PLAN_4c_renderers.md new file mode 100644 index 00000000..29d05b97 --- /dev/null +++ b/docs/PLAN_4c_renderers.md @@ -0,0 +1,171 @@ +# PLAN — Batch 4C: host-config renderers + the paired-release cutover + +> Status: **drafting** (2026-06-14). Branch `b4c-renderers` off `main`. +> Prereqs done: PR #190 (deployer, merged), #191 (`levels audit-gauges`, merged), +> #192 (#3 docroot → `/var/cache/kayak/docroot`, merging 2026-06-14 AM). +> This plan finishes Batch 4 — it replaces the hand-crafted step 5 of +> `deploy/INSTALL-paired-release.md` (branch `b4c-paired-install`) with renderers +> driven by `kayak.host.HostConfig`, adds the deployer's serving-path verification, +> and flips the live host to `SERVING_CUTOVER=yes`. + +## Goal + +The deployer (`kayak-deploy.sh`) refuses to activate until the host is fully cut +over to the `/opt/kayak/current` paired-release layout: nginx `root`, FPM +`open_basedir`, and every `levels`-running consumer unit must point at +`current`/the shared docroot. Today those artifacts are **hand-crafted** (runbook +step 5). 4C makes them **rendered from `host.yaml`** so the cutover is +reproducible and verifiable, then performs the cutover on the live host. + +## Consumer classification (verified against `systemd/` on `main`, 2026-06-14) + +The cutover only re-points units that execute **engine code from the venv**. The +`levels audit-gauges` promotion (#191) moved audit-gauges from the +"host-level/source-script" column into the engine column — the runbook §5c +(which predates #191) lists only 5; it is now **6**. + +**A. Engine consumers — re-point `ExecStart` → `/opt/kayak/current/venv/bin/levels`, +fix `DATASET_DIR`/`OUTPUT_DIR`/`WorkingDirectory`/`ReadWritePaths`:** +`kayak-pipeline`, `kayak-decimate`, `kayak-editor-retention`, `kayak-fetch-osmb`, +`kayak-status`, **`kayak-audit-gauges`** (new, via #191). + +**B. Checkout-script consumers — run `/home//kayak/{scripts,systemd}/*.sh`, +none of which invoke `levels`.** They keep running from the repo checkout (which +still exists on the live host), so the cutover does **not** re-point them. Two of +them touch the DB and so must still be in the deployer's quiesce set +(`KAYAK_UNITS`) even though they're not re-pointed (descriptions verified against +the live scripts — PR #193 review #1): +- `kayak-healthcheck` (`scripts/health-check.sh`) — reads the DB → **quiesce**. +- `kayak-config-drift` (`scripts/check-config-drift.sh`) — compares repo↔/etc, no + DB content read. +- `kayak-recap` (`systemd/kayak-recap.sh`) — runs `${KAYAK_HOME}/.venv/bin/python3 + scripts/recap.py` (the **editable-install venv**, NOT a `levels` subcommand); + journald only, no DB/dataset. Stays on the checkout — re-pointing buys nothing + (running old code to render a journald email is harmless); no quiesce. +- `kayak-heartbeat` (`systemd/kayak-heartbeat.sh`) — `mail`/msmtp heartbeat (no + curl); `stat()`s the DB file mtime (no content read, no lock) → no quiesce. +- `kayak-cert-expiry` (`scripts/check-cert-expiry.sh`), `kayak-cert-renewal-test` + (`certbot`) — no DB. + +> Note: recap uses the venv but isn't a class-A unit — it runs `python3 +> scripts/recap.py`, not `levels`, and touches neither the migrated DB nor the +> release dataset, so §Goal's "every `levels`-running consumer points at current" +> doesn't reach it. The D-CONSUMER derivation keys re-pointing off "ExecStart runs +> `…/levels`", which correctly excludes recap. + +**C. Pure host-level — exempt via `KAYAK_HOST_UNITS`:** `kayak-backup-{hourly, +weekly,offsite}` (shell), `kayak-notify-failure@` (template), `kayak-fail-test`. + +> **Open decision D-CONSUMER:** the deployer's activation gate verifies every +> `KAYAK_UNITS` member's `ExecStart` references `$ROOT/current` *unless* it's in +> `KAYAK_HOST_UNITS`. The repo-shell-script consumers (B) reference neither `current` +> nor the venv — they reference `/home//kayak`. So they must be listed in +> `KAYAK_HOST_UNITS` to pass the gate, **even though some read the DB** (so they +> must ALSO be quiesced). Today `KAYAK_UNITS` (quiesce) and `KAYAK_HOST_UNITS` +> (gate-exempt) are independent lists, which already supports "quiesce but don't +> verify". The renderer/runbook must emit a `KAYAK_HOST_UNITS` that includes the +> class-B units, and `KAYAK_UNITS` that includes the DB-touching ones. Plan: derive +> both lists from the installed `kayak-*.timer` set + this classification table, +> rather than hand-maintaining them (closes the "complete consumer enumeration" +> 4C item). + +## `HostConfig` additions (increment 1) + +New non-secret fields on `kayak.host.HostConfig` (all with current-WKCC defaults, +keep-current-then-flip — the live `host.yaml` flips them at cutover): + +| field | default (current) | cutover value | used by | +|---|---|---|---| +| `service_user` | `pat` | `pat` | unit `User=`, ACL, `KAYAK_APP_USER` | +| `service_home` | `/home/pat` | `/home/pat` | `KAYAK_HOME`, DB/var/log dirs | +| `release_root` | `/opt/kayak` | `/opt/kayak` | venv/dataset paths, `$ROOT` | +| `fpm_pool_php` | `8.4` | `8.4` | FPM pool path | +| `server_names` | (per-vhost, see below) | — | nginx `server_name` | + +`docroot` (existing field) is reused: it stays `/home/pat/public_html` until the +cutover `host.yaml` sets it to `/var/cache/kayak/docroot` (matching the deployer's +`KAYAK_DOCROOT`). Derived (not stored): venv = `{release_root}/current/venv`, +release dataset = `{release_root}/current/dataset`, FPM pool = +`/etc/php/{fpm_pool_php}/fpm/pool.d/kayak.conf`. + +The vhost `server_names` need a small structured type (a list of +`{server_name, cert_host, enabled}`) for the three sites (`levels.wkcc.org`, +`levels.mousebrains.com`, `levels-test`). That type lands with the **nginx +renderer (increment 3)** that consumes it, not here — adding it before its +consumer risks the wrong shape. Increment 1 is the scalar fields only. + +## Renderer mechanism + +A `levels` subcommand per the `emit-config` precedent (emits text; the installer/ +runbook redirects it). Proposed: +- `levels render-units [--out-dir DIR]` → the class-A drop-in files + (`.service.d/cutover.conf`) from `HostConfig`. Default: print a manifest; + `--out-dir` writes the files. +- `levels render-nginx [--site NAME]` / `levels render-fpm` → vhost root + + `open_basedir` substitutions (or a sed-spec the runbook applies). Increment 3. + +Each renderer is **pure** (HostConfig + templates → text), unit-tested by asserting +the rendered text matches the runbook §5 spec. Real `systemd-analyze verify` / +`nginx -t` validation happens on the VM (Pat-driven; see handoff). + +## The class-A drop-in spec (`.service.d/cutover.conf`) + +```ini +[Service] +ExecStart= +ExecStart=/opt/kayak/current/venv/bin/levels +Environment=DATASET_DIR=/opt/kayak/current/dataset +Environment=OUTPUT_DIR=/var/cache/kayak/docroot +WorkingDirectory=/opt/kayak/current +ReadWritePaths=/var/cache/kayak/docroot /home//DB +``` +(The leading empty `ExecStart=` resets the base unit's value — required by systemd +to replace, not append.) ` ` comes from the base unit's ExecStart +tail (e.g. `pipeline`; `status --output …`; `audit-gauges --days 16 --email +${AUDIT_EMAIL}`). `ReadWritePaths` differs per unit (status writes +`status_output`; audit-gauges writes the metadata cache — see #191). The renderer +must carry the per-unit write-path set, not a blanket one. + +## Increment sequence (each a reviewable PR; VM-validated where noted) + +1. **`HostConfig` renderer fields + tests.** Pure schema (scalar fields); no + behavior change (defaults = current). Foundational. *(merged in PR #193's branch)* +2. **`levels render-units` + tests** asserting the 6 class-A drop-ins match the + spec. Also adds the two relocatable-cache fields (`map_layers_dir`, + `gauge_metadata_cache`) — surfaced here because `fetch-osmb`/`audit-gauges` + default those dirs *relative to the install root*, which is read-only under + `/opt/kayak/current`, so the drop-ins must point them at `/var/cache/kayak/*` + (keep-current-then-flip, like `docroot`). Wires nothing into the host. *(this PR)* +3. **`levels render-nginx` / `render-fpm` + tests.** The root/`open_basedir` + substitutions. +4. **Deployer serving-path gate + quiesce-timeout fix** (`deploy/kayak-deploy.sh`): + when `SERVING_CUTOVER=yes`, verify nginx root / FPM `open_basedir` / unit + `OUTPUT_DIR`+`ReadWritePaths` resolve to `$KAYAK_DOCROOT`; back out maintenance + on a drain timeout (the [[deploy_quiesce_timeout_followup]] fix). Branch off + #192. Slow-test the gate. +5. **Derive `KAYAK_UNITS`/`KAYAK_HOST_UNITS` from installed timers** (closes the + complete-consumer-enumeration item; resolves D-CONSUMER). +6. **Runbook §5 rewrite** (`deploy/INSTALL-paired-release.md` on `b4c-paired-install`): + replace hand-crafted step 5 with the `render-*` calls; update for the #3 docroot + and the audit-gauges promotion. Rebase onto merged #192. +7. **Cutover the live host** (Pat-driven, with the VM rehearsed first): ship the + live `host.yaml`, run the renderers, ACLs, flip `SERVING_CUTOVER=yes`, first + real `kayak-deploy` activation. Flip the generic defaults last. + +## VM-validation handoff + +Increments 2–5 produce text/gate logic unit-tested here, but the rendered units/ +vhosts must pass `systemd-analyze verify` + `nginx -t` on the arm64 test VM (ssh +`levels-mac`) before the live cutover — that's the same drive→codex→Pat rehearsal +flow as [[vm_rehearsal_plan]]. The hand-crafted artifacts already validated on the +VM (2026-06-13) are the renderers' golden output to diff against. + +## Reproduce / cross-check + +```bash +# consumer classification: +for f in systemd/kayak-*.service; do printf '%s\t%s\n' "$(basename "$f")" \ + "$(grep -h '^ExecStart=' "$f" | head -1)"; done +# the hand-crafted spec the renderers must reproduce: +git show b4c-paired-install:deploy/INSTALL-paired-release.md # § step 5 +``` diff --git a/src/kayak/cli/main.py b/src/kayak/cli/main.py index 1415caeb..3333bbde 100644 --- a/src/kayak/cli/main.py +++ b/src/kayak/cli/main.py @@ -27,6 +27,7 @@ orphan_check, pipeline, recover_metadata, + render_units, seed_maintainer, status, sync_metadata, @@ -75,6 +76,7 @@ def main() -> None: analyze_logs.addArgs(subparsers) status.addArgs(subparsers) sync_metadata.addArgs(subparsers) + render_units.addArgs(subparsers) import_metadata.addArgs(subparsers) recover_metadata.addArgs(subparsers) generate_sources.addArgs(subparsers) diff --git a/src/kayak/cli/render_units.py b/src/kayak/cli/render_units.py new file mode 100644 index 00000000..04efeca1 --- /dev/null +++ b/src/kayak/cli/render_units.py @@ -0,0 +1,76 @@ +"""``levels render-units`` — emit the Batch 4C cutover systemd drop-ins. + +Reads ``host.yaml`` and prints (or writes) one +``.service.d/cutover.conf`` per engine consumer — the drop-ins that +re-point each ``levels``-running unit at the ``/opt/kayak/current`` release +venv. Mirrors ``emit-config``: the tool emits text; the install runbook / +deployer applies it. See ``docs/PLAN_4c_renderers.md``. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from kayak.host import load_host_config +from kayak.host_render import render_cutover_dropins + + +def addArgs(subparsers: argparse._SubParsersAction[argparse.ArgumentParser]) -> None: + p = subparsers.add_parser( + "render-units", + help="Render the paired-release cutover systemd drop-ins from host.yaml (4C)", + ) + p.set_defaults(func=render_units) + p.add_argument( + "--out-dir", + type=Path, + help="Write .service.d/cutover.conf trees under this dir " + "(e.g. /etc/systemd/system); default: print a manifest to stdout", + ) + p.add_argument( + "--host-config", + type=Path, + help="host.yaml path (default: $KAYAK_HOST_CONFIG or /etc/kayak/host.yaml)", + ) + + +def render_units(args: argparse.Namespace) -> int: + try: + host = load_host_config(args.host_config) + except ValueError as e: + print(f"render-units: host config invalid: {e}", file=sys.stderr) + return 1 + + dropins = render_cutover_dropins(host) + + if args.out_dir is None: + for d in dropins: + print(f"# ==> {d.path}") + print(d.text) + return 0 + + # Every drop-in references {release_root}/current (DATASET_DIR, WorkingDirectory, + # the ExecStart venv), which exists only AFTER the first paired-release + # activation. Installing them onto a host whose release isn't live yet would + # point all six consumers at a non-existent dir and break them, so warn — the + # cutover runbook installs these only once `current` resolves (PR #193 review #3). + current = Path(host.release_root) / "current" + if not current.exists(): + print( + f"render-units: WARNING: {current} does not exist yet — these drop-ins " + "reference it and will break the consumers if applied before the first " + "paired-release activation (cutover order: stage+activate, THEN install).", + file=sys.stderr, + ) + + # NOTE: this writes cutover.conf for the current 6-unit set but does not sweep a + # stale drop-in if that set ever shrinks; revisit if a consumer is retired. + out_dir: Path = args.out_dir + for d in dropins: + dest = out_dir / d.path + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text(d.text, encoding="utf-8") + print(f"wrote {dest}") + return 0 diff --git a/src/kayak/host.py b/src/kayak/host.py index 880e7108..2315011e 100644 --- a/src/kayak/host.py +++ b/src/kayak/host.py @@ -36,10 +36,25 @@ HOST_YAML = "/etc/kayak/host.yaml" -_HOSTNAME_RE = re.compile(r"^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+$") +# All anchored with \A…\Z, NOT ^…$: Python's `$` also matches just before a +# trailing newline, so `^…$` would accept e.g. "pat\n" and let a newline smuggle +# a second directive into a rendered unit / shell command (PR #193 review #2). +_HOSTNAME_RE = re.compile(r"\A[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+\Z") # rclone remote names: word characters and hyphens (no ':' — the colon is # syntax, appended by consumers). -_RCLONE_REMOTE_RE = re.compile(r"^[A-Za-z0-9_-]+$") +_RCLONE_REMOTE_RE = re.compile(r"\A[A-Za-z0-9_-]+\Z") +# POSIX-portable service account name (useradd's NAME_REGEX): start with a +# lower-case letter or underscore, then lower/digit/underscore/hyphen. +_USERNAME_RE = re.compile(r"\A[a-z_][a-z0-9_-]*\Z") +# PHP-FPM pool version as it appears in the /etc/php//fpm path (major.minor). +# [0-9] not \d — \d also matches Unicode digits, which the path can't contain. +_PHP_VERSION_RE = re.compile(r"\A[0-9]+\.[0-9]+\Z") +# Whitespace / control chars in a path field. The renderers interpolate paths +# into systemd directives by plain f-string (Environment=, ReadWritePaths=, +# WorkingDirectory=, ExecStart=); a newline injects a directive and a space +# splits a ReadWritePaths= entry, so reject both even though host.yaml is trusted +# (defense-in-depth, the same reasoning that hardened service_user — review #2). +_PATH_BAD_CHAR_RE = re.compile(r"[\s\x00-\x1f\x7f]") class HostConfig(BaseModel): @@ -58,6 +73,34 @@ class HostConfig(BaseModel): docroot: str = "/home/pat/public_html" cert_host: str = "levels.wkcc.org" + # --- paired-release cutover / renderers (4C) --- + # The service account + its home, the paired-release root, and the PHP-FPM + # pool version the systemd-unit / nginx-vhost / FPM renderers need. Defaults + # are the current WKCC values (keep-current-then-flip): a host with no + # host.yaml renders the live shape, and the cutover host.yaml flips them. + # ``docroot`` (above) is reused — it stays ``public_html`` until the cutover + # host.yaml sets it to ``/var/cache/kayak/docroot`` (= the deployer's + # ``KAYAK_DOCROOT``). Paths the renderers derive, NOT stored: the release venv + # (``{release_root}/current/venv``), the release dataset + # (``{release_root}/current/dataset``), and the FPM pool + # (``/etc/php/{fpm_pool_php}/fpm/pool.d/kayak.conf``). See + # docs/PLAN_4c_renderers.md. + service_user: str = "pat" + service_home: str = "/home/pat" + release_root: str = "/opt/kayak" + fpm_pool_php: str = "8.4" + # Generated-runtime-data dirs that the engine defaults resolve RELATIVE to the + # install root (config.py BASE_DIR): fetch-osmb's map-layer staging and the + # gauge-audit metadata cache. In the editable install they land under the repo + # checkout; under an immutable /opt/kayak/current release that root is + # read-only, so the cutover unit drop-ins must point these at stable writable + # cache paths. Keep-current defaults (the live repo-relative locations); the + # cutover host.yaml flips them to /var/cache/kayak/* (regenerable cache, + # alongside the #3 docroot). map_layers_dir is a dir; gauge_metadata_cache is + # the sqlite FILE. + map_layers_dir: str = "/home/pat/kayak/var/osmb" + gauge_metadata_cache: str = "/home/pat/kayak/Gauge-metadata-cache/gauges.db" + # --- backup policy (S8) --- backup_dir: str = "/home/pat/backups" offsite_remote: str = "gdrive-crypt" @@ -79,11 +122,41 @@ def _tz(cls, v: str) -> str: raise ValueError(f"must be an IANA timezone name (got {v!r})") from e return v - @field_validator("nginx_log_glob", "status_output", "docroot", "backup_dir") + @field_validator( + "nginx_log_glob", + "status_output", + "docroot", + "backup_dir", + "service_home", + "release_root", + "map_layers_dir", + "gauge_metadata_cache", + ) @classmethod def _abs_path(cls, v: str) -> str: if not v.startswith("/"): raise ValueError(f"must be an absolute path (got {v!r})") + if _PATH_BAD_CHAR_RE.search(v): + raise ValueError(f"must not contain whitespace or control characters (got {v!r})") + return v + + @field_validator("service_user") + @classmethod + def _username(cls, v: str) -> str: + # POSIX-portable service account name: lower/digit/underscore/hyphen, + # not starting with a hyphen. Renderers interpolate it into unit User= + # and shell ACL commands, so reject anything that isn't a bare name. + if not _USERNAME_RE.match(v): + raise ValueError(f"must be a bare POSIX username (got {v!r})") + return v + + @field_validator("fpm_pool_php") + @classmethod + def _php_version(cls, v: str) -> str: + # PHP-FPM pool version as it appears in /etc/php//fpm — major.minor + # only (Debian packages php8.4, never a patch level in the path). + if not _PHP_VERSION_RE.match(v): + raise ValueError(f"must be a major.minor PHP version like '8.4' (got {v!r})") return v @field_validator("cert_host") diff --git a/src/kayak/host_render.py b/src/kayak/host_render.py new file mode 100644 index 00000000..dfbfea86 --- /dev/null +++ b/src/kayak/host_render.py @@ -0,0 +1,110 @@ +"""Render the paired-release cutover systemd drop-ins from ``HostConfig`` (4C). + +Each **engine consumer** — the units that run ``levels …`` from the venv — gets a +``.service.d/cutover.conf`` drop-in that re-points ``ExecStart`` at the +release venv and overrides the paths that move off the (now immutable) install +root onto the persistent home / shared cache. The repo-shell-script consumers and +the host-level units are NOT re-pointed (they don't run the venv); see +``docs/PLAN_4c_renderers.md`` for the full classification. + +Pure: ``HostConfig`` → text. The CLI wrapper (``levels render-units``) writes or +prints the result; ``systemd-analyze verify`` of the merged units happens on the +VM, not here. + +Drop-in semantics that the templates rely on: +- ``ExecStart=`` (empty) RESETS the base unit's value; the next ``ExecStart=`` + sets the venv binary + the same subcommand/args. Without the reset, a oneshot + would run both the old and new commands. +- ``ReadWritePaths=`` (empty) RESETS the accumulated list; the next assignment is + the exact cutover writable set (e.g. the shared docroot replaces ``public_html``). +- ``Environment=`` assignments override the same key from the unit's + ``EnvironmentFile=`` because the drop-in is merged after the base unit. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import PurePosixPath + +from kayak.host import HostConfig + + +@dataclass(frozen=True) +class CutoverDropIn: + """One engine consumer's rendered ``cutover.conf`` drop-in.""" + + unit: str # systemd unit, e.g. "kayak-pipeline.service" + text: str # the drop-in file content + + @property + def path(self) -> str: + """Install path relative to the systemd unit dir.""" + return f"{self.unit}.d/cutover.conf" + + +def _parent_dir(file_path: str) -> str: + """The directory a file lives in — what ``ReadWritePaths=`` must grant.""" + return str(PurePosixPath(file_path).parent) + + +def render_cutover_dropins(h: HostConfig) -> list[CutoverDropIn]: + """Render the cutover drop-in for every engine consumer, in install order. + + The six units that execute ``levels`` from the venv (the audit-gauges one + joined this set when #191 promoted the source script to a subcommand). + """ + venv = f"{h.release_root}/current/venv/bin/levels" + current = f"{h.release_root}/current" + dataset = f"{current}/dataset" + db = f"{h.service_home}/DB" + + # (unit, ExecStart args after the binary, extra Environment=, ReadWritePaths) + # DATASET_DIR is pinned to the RELEASE's snapshot on every unit (not the live + # kayak_data clone) so a consumer always reads the dataset that shipped with + # the running code; harmless on the units that don't read it. + specs: list[tuple[str, list[str], dict[str, str], list[str]]] = [ + # pipeline builds the docroot → OUTPUT_DIR + the shared docroot RWP. + ("kayak-pipeline", ["pipeline"], {"OUTPUT_DIR": h.docroot}, [h.docroot, db]), + ("kayak-decimate", ["decimate"], {}, [db]), + ("kayak-editor-retention", ["editor-retention"], {}, [db]), + # fetch-osmb stages map layers into a dir the engine default resolves + # relative to the install root → relocate to the persistent map-layers dir. + ( + "kayak-fetch-osmb", + ["fetch-osmb"], + {"MAP_LAYERS_DIR": h.map_layers_dir}, + [h.map_layers_dir], + ), + # status writes its --output page (under the home, persists across releases). + ( + "kayak-status", + ["status", "--output", h.status_output], + {}, + [_parent_dir(h.status_output)], + ), + # audit-gauges reads the DB + writes the gauge-metadata cache, which also + # defaults relative to the install root → relocate it (same trap as #191). + ( + "kayak-audit-gauges", + ["audit-gauges", "--days", "16", "--email", "${AUDIT_EMAIL}"], + {"GAUGE_METADATA_CACHE": h.gauge_metadata_cache}, + [db, _parent_dir(h.gauge_metadata_cache)], + ), + ] + + dropins: list[CutoverDropIn] = [] + for name, args, extra_env, write_paths in specs: + env = {"DATASET_DIR": dataset, **extra_env} + lines = [ + "# Generated by `levels render-units` (Batch 4C cutover). Do not edit;", + "# re-render from host.yaml. See docs/PLAN_4c_renderers.md.", + "[Service]", + "ExecStart=", + f"ExecStart={venv} {' '.join(args)}", + ] + lines += [f"Environment={k}={v}" for k, v in env.items()] + lines.append(f"WorkingDirectory={current}") + lines.append("ReadWritePaths=") + lines.append(f"ReadWritePaths={' '.join(write_paths)}") + dropins.append(CutoverDropIn(unit=f"{name}.service", text="\n".join(lines) + "\n")) + return dropins diff --git a/tests/test_host_config.py b/tests/test_host_config.py index b579296d..5234b398 100644 --- a/tests/test_host_config.py +++ b/tests/test_host_config.py @@ -24,6 +24,13 @@ def test_absent_returns_current_defaults(self, tmp_path: Path) -> None: assert c.offsite_remote == "gdrive-crypt" assert c.offsite_keep == 26 assert c.offsite_label == "rclone → gdrive-crypt:" + # 4C renderer fields default to the current WKCC shape. + assert c.service_user == "pat" + assert c.service_home == "/home/pat" + assert c.release_root == "/opt/kayak" + assert c.fpm_pool_php == "8.4" + assert c.map_layers_dir == "/home/pat/kayak/var/osmb" + assert c.gauge_metadata_cache == "/home/pat/kayak/Gauge-metadata-cache/gauges.db" def test_overrides_applied_partially(self, tmp_path: Path) -> None: f = tmp_path / "host.yaml" @@ -85,6 +92,64 @@ def test_zero_offsite_keep_rejected(self, tmp_path: Path) -> None: with pytest.raises(ValueError, match="at least 1"): host.load_host_config(f) + def test_renderer_fields_override(self, tmp_path: Path) -> None: + f = tmp_path / "host.yaml" + f.write_text( + "service_user: kayak\n" + "service_home: /srv/kayak\n" + "release_root: /srv/kayak/releases\n" + "fpm_pool_php: '8.3'\n" + "docroot: /var/cache/kayak/docroot\n" + "map_layers_dir: /var/cache/kayak/map-layers\n" + "gauge_metadata_cache: /var/cache/kayak/gauge-metadata/gauges.db\n" + ) + c = host.load_host_config(f) + assert c.service_user == "kayak" + assert c.service_home == "/srv/kayak" + assert c.release_root == "/srv/kayak/releases" + assert c.fpm_pool_php == "8.3" + assert c.docroot == "/var/cache/kayak/docroot" + assert c.map_layers_dir == "/var/cache/kayak/map-layers" + assert c.gauge_metadata_cache == "/var/cache/kayak/gauge-metadata/gauges.db" + + def test_bad_service_user_rejected(self, tmp_path: Path) -> None: + # Renderers interpolate it into unit User= and shell ACL commands. + f = tmp_path / "host.yaml" + f.write_text("service_user: 'pat; rm -rf /'\n") + with pytest.raises(ValueError, match="POSIX username"): + host.load_host_config(f) + + def test_relative_release_root_rejected(self, tmp_path: Path) -> None: + f = tmp_path / "host.yaml" + f.write_text("release_root: opt/kayak\n") + with pytest.raises(ValueError, match="absolute"): + host.load_host_config(f) + + def test_bad_fpm_version_rejected(self, tmp_path: Path) -> None: + f = tmp_path / "host.yaml" + f.write_text("fpm_pool_php: '8.4.3'\n") + with pytest.raises(ValueError, match=r"major\.minor"): + host.load_host_config(f) + + def test_service_user_trailing_newline_rejected(self) -> None: + # `\Z` not `$`: a trailing newline would otherwise smuggle a second + # directive into a rendered unit / ACL command (PR #193 review #2). + with pytest.raises(ValueError, match="POSIX username"): + host.HostConfig(service_user="pat\n") + + def test_fpm_version_unicode_digit_rejected(self) -> None: + # `[0-9]` not `\d` — \d matches Unicode digits the path can't contain. + with pytest.raises(ValueError, match=r"major\.minor"): + host.HostConfig(fpm_pool_php="8.٤") # Arabic-Indic 4 + + def test_path_with_whitespace_rejected(self) -> None: + # Path fields are f-string-interpolated into systemd directives; a space + # splits a ReadWritePaths= entry, a newline injects a directive. + with pytest.raises(ValueError, match="whitespace or control"): + host.HostConfig(docroot="/var/cache/kayak/docroot\nExecStartPre=/bin/x") + with pytest.raises(ValueError, match="whitespace or control"): + host.HostConfig(release_root="/opt/kayak extra") + def test_malformed_yaml_fails_closed(self, tmp_path: Path) -> None: f = tmp_path / "host.yaml" f.write_text("timezone: [unclosed\n") diff --git a/tests/test_host_render.py b/tests/test_host_render.py new file mode 100644 index 00000000..de4edfbf --- /dev/null +++ b/tests/test_host_render.py @@ -0,0 +1,159 @@ +"""Tests for kayak.host_render — the 4C cutover systemd drop-in renderer.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +from kayak.host import HostConfig +from kayak.host_render import render_cutover_dropins + + +def _cutover_host() -> HostConfig: + """A fully cut-over host.yaml shape (paths flipped off the install root).""" + return HostConfig( + service_user="pat", + service_home="/home/pat", + release_root="/opt/kayak", + docroot="/var/cache/kayak/docroot", + status_output="/home/pat/var/status.html", + map_layers_dir="/var/cache/kayak/map-layers", + gauge_metadata_cache="/var/cache/kayak/gauge-metadata/gauges.db", + ) + + +def _by_unit(h: HostConfig) -> dict[str, str]: + return {d.unit: d.text for d in render_cutover_dropins(h)} + + +class TestRenderCutoverDropins: + def test_exactly_the_six_engine_consumers(self) -> None: + units = {d.unit for d in render_cutover_dropins(_cutover_host())} + assert units == { + "kayak-pipeline.service", + "kayak-decimate.service", + "kayak-editor-retention.service", + "kayak-fetch-osmb.service", + "kayak-status.service", + "kayak-audit-gauges.service", # the 6th, via #191's promotion + } + + def test_dropin_path_is_the_systemd_override(self) -> None: + d = render_cutover_dropins(_cutover_host())[0] + assert d.path == f"{d.unit}.d/cutover.conf" + + def test_every_dropin_resets_then_repoints_execstart_at_the_release_venv(self) -> None: + for unit, text in _by_unit(_cutover_host()).items(): + # The empty ExecStart= reset must precede the new one (oneshot would + # otherwise run both the old and new commands). + assert "\nExecStart=\nExecStart=/opt/kayak/current/venv/bin/levels " in text, unit + assert "/home/pat/.venv/bin/levels" not in text, unit + + def test_every_dropin_resets_then_sets_readwritepaths(self) -> None: + for unit, text in _by_unit(_cutover_host()).items(): + assert "\nReadWritePaths=\nReadWritePaths=" in text, unit + + def test_dataset_dir_pinned_to_release_on_every_unit(self) -> None: + for unit, text in _by_unit(_cutover_host()).items(): + assert "Environment=DATASET_DIR=/opt/kayak/current/dataset" in text, unit + + def test_pipeline_builds_the_shared_docroot(self) -> None: + t = _by_unit(_cutover_host())["kayak-pipeline.service"] + assert "ExecStart=/opt/kayak/current/venv/bin/levels pipeline\n" in t + assert "Environment=OUTPUT_DIR=/var/cache/kayak/docroot" in t + assert "ReadWritePaths=/var/cache/kayak/docroot /home/pat/DB" in t + + def test_fetch_osmb_relocates_map_layers_off_the_release(self) -> None: + t = _by_unit(_cutover_host())["kayak-fetch-osmb.service"] + assert "Environment=MAP_LAYERS_DIR=/var/cache/kayak/map-layers" in t + assert "ReadWritePaths=/var/cache/kayak/map-layers" in t + + def test_audit_gauges_relocates_cache_and_keeps_email_var(self) -> None: + t = _by_unit(_cutover_host())["kayak-audit-gauges.service"] + assert ( + "ExecStart=/opt/kayak/current/venv/bin/levels " + "audit-gauges --days 16 --email ${AUDIT_EMAIL}\n" in t + ) + # The cache file relocates; ReadWritePaths grants its parent DIR + the DB. + assert "Environment=GAUGE_METADATA_CACHE=/var/cache/kayak/gauge-metadata/gauges.db" in t + assert "ReadWritePaths=/home/pat/DB /var/cache/kayak/gauge-metadata" in t + + def test_status_passes_output_arg_and_grants_its_dir(self) -> None: + t = _by_unit(_cutover_host())["kayak-status.service"] + assert ( + "ExecStart=/opt/kayak/current/venv/bin/levels status --output /home/pat/var/status.html\n" + in t + ) + assert "ReadWritePaths=/home/pat/var" in t + + def test_db_only_units_grant_only_the_db(self) -> None: + units = _by_unit(_cutover_host()) + for u in ("kayak-decimate.service", "kayak-editor-retention.service"): + assert "ReadWritePaths=\nReadWritePaths=/home/pat/DB\n" in units[u], u + assert "OUTPUT_DIR" not in units[u], u + + def test_defaults_render_the_current_live_shape(self) -> None: + # With no host.yaml override the renderer still points at /opt/kayak/current + # (release_root default) but the relocatable caches keep their live + # repo-relative values — keep-current-then-flip. + t = _by_unit(HostConfig())["kayak-fetch-osmb.service"] + assert "Environment=MAP_LAYERS_DIR=/home/pat/kayak/var/osmb" in t + g = _by_unit(HostConfig())["kayak-audit-gauges.service"] + assert ( + "Environment=GAUGE_METADATA_CACHE=/home/pat/kayak/Gauge-metadata-cache/gauges.db" in g + ) + + def test_alternate_service_user_and_home_flow_through(self) -> None: + h = HostConfig(service_home="/srv/kayak", release_root="/srv/releases") + t = _by_unit(h)["kayak-decimate.service"] + assert "ExecStart=/srv/releases/current/venv/bin/levels decimate\n" in t + assert "ReadWritePaths=/srv/kayak/DB" in t + + +class TestRenderUnitsCli: + def test_writes_dropin_files(self, tmp_path: Path) -> None: + from kayak.cli import render_units as cli + + rc = cli.render_units(argparse.Namespace(out_dir=tmp_path, host_config=None)) + assert rc == 0 + written = {p.relative_to(tmp_path).as_posix() for p in tmp_path.rglob("*") if p.is_file()} + assert "kayak-pipeline.service.d/cutover.conf" in written + assert len(written) == 6 + body = (tmp_path / "kayak-pipeline.service.d/cutover.conf").read_text() + assert "[Service]" in body and "ExecStart=" in body + + def test_manifest_to_stdout_when_no_out_dir(self, capsys) -> None: + from kayak.cli import render_units as cli + + rc = cli.render_units(argparse.Namespace(out_dir=None, host_config=None)) + assert rc == 0 + out = capsys.readouterr().out + assert "# ==> kayak-pipeline.service.d/cutover.conf" in out + + def test_malformed_host_config_is_clean_error(self, tmp_path: Path, capsys) -> None: + from kayak.cli import render_units as cli + + bad = tmp_path / "host.yaml" + bad.write_text("release_root: not-absolute\n") + rc = cli.render_units(argparse.Namespace(out_dir=None, host_config=bad)) + assert rc == 1 + assert "host config invalid" in capsys.readouterr().err + + def test_warns_when_release_current_missing(self, tmp_path: Path, capsys) -> None: + from kayak.cli import render_units as cli + + hy = tmp_path / "host.yaml" + hy.write_text(f"release_root: {tmp_path}/opt\n") # no opt/current + rc = cli.render_units(argparse.Namespace(out_dir=tmp_path / "out", host_config=hy)) + assert rc == 0 + assert "does not exist yet" in capsys.readouterr().err + + def test_no_warning_when_release_current_exists(self, tmp_path: Path, capsys) -> None: + from kayak.cli import render_units as cli + + (tmp_path / "opt" / "current").mkdir(parents=True) + hy = tmp_path / "host.yaml" + hy.write_text(f"release_root: {tmp_path}/opt\n") + rc = cli.render_units(argparse.Namespace(out_dir=tmp_path / "out", host_config=hy)) + assert rc == 0 + assert "does not exist yet" not in capsys.readouterr().err