From 7a176ea238ba81fc0e1adc8a8679838f24d2a981 Mon Sep 17 00:00:00 2001 From: Luna Date: Tue, 16 Jun 2026 04:05:12 -0700 Subject: [PATCH] Add stable v1 schema IDs and docs --- CLAUDE.md | 5 +- README.md | 2 + docs/COMMAND_REFERENCE.md | 4 + docs/ROADMAP.md | 4 +- docs/SCHEMAS.md | 126 +++++++++++++++++++ docs/SPECIFICATION.md | 5 + enodia_sentinel/cli.py | 8 +- enodia_sentinel/incident.py | 10 +- enodia_sentinel/respond.py | 4 +- enodia_sentinel/schemas.py | 16 +++ enodia_sentinel/snapshot.py | 2 + enodia_sentinel/web.py | 9 +- tests/test_schema_contracts.py | 222 +++++++++++++++++++++++++++++++++ 13 files changed, 409 insertions(+), 8 deletions(-) create mode 100644 docs/SCHEMAS.md create mode 100644 enodia_sentinel/schemas.py create mode 100644 tests/test_schema_contracts.py diff --git a/CLAUDE.md b/CLAUDE.md index 44f162e..bb057ce 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -74,8 +74,8 @@ Near-term open work: event-only and future correlation SIDs. - More event sources and correlation across exec, network, persistence, FIM, and rootcheck signals. -- Stable schema compatibility tests for alerts, incidents, status, response - plans, and response audit records. +- Stable v1 schema IDs and compatibility tests for alerts, incidents, status, + response plans, and response audit records. Start with: @@ -178,4 +178,5 @@ When changing behavior, update the relevant set: - `docs/SPECIFICATION.md` for product/data-model changes. - `docs/THREAT_MODEL.md` for trust-boundary/security-model changes. - `docs/ROADMAP.md` for planned or completed roadmap movement. +- `docs/SCHEMAS.md` for stable JSON contracts. - `config/enodia-sentinel.toml` for config knobs. diff --git a/README.md b/README.md index 1fffdf6..f5a79ce 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,8 @@ Project docs: response, fleet, and assurance layers. - [Operations guide](docs/OPERATIONS.md) — install checks, health checks, alert workflow, baseline hygiene, and evidence export. +- [JSON schemas](docs/SCHEMAS.md) — stable v1 contracts for alerts, incidents, + status, response plans, and response audit records. - [IR runbooks](docs/RUNBOOKS.md) — confirm/preserve/contain/recover playbooks per alert: reverse shells, persistence, trojaned binaries, rootkits, tampering. - [Rule reference](docs/RULES.md) — generated event-rule documentation: SID, diff --git a/docs/COMMAND_REFERENCE.md b/docs/COMMAND_REFERENCE.md index de852d7..6edd85e 100644 --- a/docs/COMMAND_REFERENCE.md +++ b/docs/COMMAND_REFERENCE.md @@ -308,6 +308,10 @@ staleness, total alerts with per-severity counts, last alert time, and eBPF monitor state. `--json` emits the same data as the dashboard `/api/status` endpoint, for cron/monitoring. +Stable JSON contracts are documented in [SCHEMAS.md](SCHEMAS.md). Current v1 +IDs cover alert objects/snapshots, incident records/views/exports, status, +response plans, and response audit JSONL records. + Exit code: - `0`: daemon is running and the heartbeat is fresh. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 47207ad..f48148f 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -89,9 +89,9 @@ Exit criteria: Purpose: define a stable single-host product. -- Commit to stable JSON schemas for alerts, incidents, status, and response +- ✅ Commit to stable JSON schemas for alerts, incidents, status, and response audits. -- Add compatibility tests for schema evolution. +- ✅ Add compatibility tests for schema evolution. - Add documentation versioning and manpage-style command reference. - Harden packaging for Arch first, then document Debian/RPM install paths. - Add signed release artifacts and checksums. diff --git a/docs/SCHEMAS.md b/docs/SCHEMAS.md new file mode 100644 index 0000000..e2bf790 --- /dev/null +++ b/docs/SCHEMAS.md @@ -0,0 +1,126 @@ +# Enodia Sentinel JSON Schemas + +Sentinel uses stable schema IDs for operator-facing JSON contracts. These IDs +are lightweight compatibility markers, not external JSON Schema files. The v1 +rule is additive: future releases may add fields, but required v1 fields should +not be removed, renamed, or change type. + +Schema IDs live in `enodia_sentinel/schemas.py`, and compatibility tests in +`tests/test_schema_contracts.py` pin the required v1 fields. + +## `enodia.alert.v1` + +Single detection object, used inside alert snapshots and by commands such as +`posture check --json`. + +Required fields: + +| Field | Type | Meaning | +|---|---|---| +| `sid` | integer | Stable signature/rule id. | +| `severity` | string | `MEDIUM`, `HIGH`, or `CRITICAL`. | +| `signature` | string | Stable machine-readable detection name. | +| `classtype` | string | Snort/Suricata-style category. | +| `key` | string | Cooldown/dedup identity. | +| `detail` | string | Human-readable finding detail. | +| `pids` | array | Process IDs tied to the finding. | + +## `enodia.alert.snapshot.v1` + +Structured alert sidecar written beside each text snapshot under `log_dir`. + +Required fields: + +| Field | Type | Meaning | +|---|---|---| +| `schema` | string | `enodia.alert.snapshot.v1`. | +| `time` | string | ISO-8601 capture time. | +| `host` | string | Hostname at capture time. | +| `severity` | string | Highest severity in the snapshot. | +| `incident_id` | string or null | Incident grouping id, when enabled. | +| `alerts` | array | `enodia.alert.v1` objects. | +| `processes` | array | Captured process context for alert PIDs. | + +## `enodia.incident.v1` + +Incident index record stored in `incidents.json`. + +Required fields: + +| Field | Type | Meaning | +|---|---|---| +| `schema` | string | `enodia.incident.v1`. | +| `id` | string | Stable incident id. | +| `host` | string | Host that produced the incident. | +| `first_ts`, `last_ts` | number | Unix timestamps for first/last activity. | +| `first_seen`, `last_seen` | string | ISO-8601 first/last activity times. | +| `severity` | string | Highest incident severity. | +| `signatures` | array | Unique detection signatures in the incident. | +| `sids` | array | Unique SIDs in the incident. | +| `pids` | array | Alert PIDs observed in the incident. | +| `lineage` | array | Process-lineage IDs used for grouping. | +| `snapshots` | array | Member snapshot filenames. | +| `alert_count` | integer | Total alerts grouped into the incident. | + +`incident show --json` returns `enodia.incident.view.v1` with `schema`, +`incident`, and `timeline`. The dashboard incident API returns the same schema +plus `snapshots`. `incident export` returns `enodia.incident.bundle.v1` with +`schema`, `incident`, and inlined `snapshots`. + +## `enodia.status.v1` + +Health/status object returned by `status --json` and `/api/status`. + +Required fields: + +| Field | Type | Meaning | +|---|---|---| +| `schema` | string | `enodia.status.v1`. | +| `version` | string | Sentinel package version. | +| `running` | boolean | Whether the daemon pid appears alive. | +| `total_alerts` | integer | Count of retained alert snapshots. | +| `counts` | object | Per-severity retained alert counts. | +| `last_alert` | string or null | Most recent retained alert time. | +| `ebpf`, `ebpf_exec`, `ebpf_syscall` | string | Last observed sensor states. | +| `host` | string | Hostname. | +| `heartbeat_age` | number or null | Seconds since heartbeat, if present. | +| `heartbeat_stale` | boolean | Whether heartbeat exceeds configured max age. | + +## `enodia.response.plan.v1` + +Dry-run response plan from `respond plan `. + +Required fields: + +| Field | Type | Meaning | +|---|---|---| +| `schema` | string | `enodia.response.plan.v1`. | +| `plan_id` | string | Stable plan id for the incident. | +| `incident_id` | string | Source incident. | +| `created_at` | string | ISO-8601 plan time. | +| `mode` | string | Currently `dry-run`. | +| `apply_supported` | boolean | Currently `false`. | +| `summary` | object | Severity, signatures, snapshot/action counts. | +| `actions` | array | Reviewable action proposals. | +| `notes` | array | Operator cautions. | + +CLI-generated plans may also include `artifacts` with saved plan/audit paths. + +## `enodia.response.audit.v1` + +JSONL record appended when the CLI persists a response plan. + +Required fields: + +| Field | Type | Meaning | +|---|---|---| +| `schema` | string | `enodia.response.audit.v1`. | +| `time` | string | ISO-8601 audit time. | +| `event` | string | Currently `response_plan_generated`. | +| `actor` | string | Caller identity supplied by the CLI/workflow. | +| `plan_id` | string | Persisted plan id. | +| `incident_id` | string | Source incident id. | +| `mode` | string | Plan mode. | +| `apply_supported` | boolean | Whether the plan supports apply. | +| `action_count` | integer | Number of proposed actions. | +| `plan_path` | string | Persisted plan artifact path. | diff --git a/docs/SPECIFICATION.md b/docs/SPECIFICATION.md index ab87940..322f7e1 100644 --- a/docs/SPECIFICATION.md +++ b/docs/SPECIFICATION.md @@ -190,6 +190,11 @@ schema. Grouping is process-lineage first, with a time-window fallback for PID-less alerts; see the [command reference](COMMAND_REFERENCE.md) and [roadmap](ROADMAP.md). +Operator-facing JSON contracts carry stable v1 schema IDs for alert snapshots, +incident records/views/exports, status, response plans, and response audit +records. The schema reference is [SCHEMAS.md](SCHEMAS.md); compatibility tests +pin required v1 fields and allow additive evolution. + ## Configuration Model Configuration is a TOML file loaded at startup. Unknown keys are ignored so diff --git a/enodia_sentinel/cli.py b/enodia_sentinel/cli.py index 64e71c7..058a695 100644 --- a/enodia_sentinel/cli.py +++ b/enodia_sentinel/cli.py @@ -12,6 +12,7 @@ import signal import sys from . import __version__, detectors +from . import schemas from .config import Config from .daemon import Sentinel from .system import SystemState, scan_suid_binaries @@ -376,6 +377,7 @@ def _cmd_incident(cfg: Config, action: str, iid: str | None, as_json: bool) -> i if action == "export": bundle = { + "schema": schemas.INCIDENT_BUNDLE_V1, "incident": inc, "snapshots": [r for name in inc.get("snapshots", []) if (r := _load_snapshot_report(cfg, name)) is not None], @@ -386,7 +388,11 @@ def _cmd_incident(cfg: Config, action: str, iid: str | None, as_json: bool) -> i # action == "show" timeline = _incident_timeline(cfg, inc) if as_json: - print(json.dumps({"incident": inc, "timeline": timeline}, indent=2)) + print(json.dumps({ + "schema": schemas.INCIDENT_VIEW_V1, + "incident": inc, + "timeline": timeline, + }, indent=2)) return 0 print(f"Incident {inc['id']} [{inc.get('severity', '?')}] on {inc.get('host', '?')}") print(f" first seen: {inc.get('first_seen', '?')}") diff --git a/enodia_sentinel/incident.py b/enodia_sentinel/incident.py index e7c208b..80a08fc 100644 --- a/enodia_sentinel/incident.py +++ b/enodia_sentinel/incident.py @@ -28,6 +28,7 @@ import time from collections.abc import Callable, Iterable from datetime import datetime +from . import schemas from .alert import Alert, Severity from .config import Config @@ -112,7 +113,12 @@ def index_path(cfg: Config): def load_index(cfg: Config) -> dict: try: data = json.loads(index_path(cfg).read_text()) - return data if isinstance(data, dict) else {} + if not isinstance(data, dict): + return {} + for inc in data.values(): + if isinstance(inc, dict): + inc.setdefault("schema", schemas.INCIDENT_V1) + return data except (OSError, ValueError): return {} @@ -161,6 +167,7 @@ def record(cfg: Config, snapshot_name: str, alerts: list[Alert], if iid is None: iid = _new_id(when) index[iid] = { + "schema": schemas.INCIDENT_V1, "id": iid, "host": host, "first_ts": when, "last_ts": when, "first_seen": iso, "last_seen": iso, @@ -169,6 +176,7 @@ def record(cfg: Config, snapshot_name: str, alerts: list[Alert], "lineage": [], "snapshots": [], "alert_count": 0, } inc = index[iid] + inc.setdefault("schema", schemas.INCIDENT_V1) inc["last_ts"] = when inc["last_seen"] = iso inc["severity"] = max_severity(inc.get("severity", severity), severity) diff --git a/enodia_sentinel/respond.py b/enodia_sentinel/respond.py index 980fe15..d38c711 100644 --- a/enodia_sentinel/respond.py +++ b/enodia_sentinel/respond.py @@ -15,6 +15,7 @@ from datetime import datetime from pathlib import Path from typing import Any +from . import schemas from .config import Config from .netutil import is_public_ip @@ -189,7 +190,7 @@ def build_plan(bundle: dict, cfg: Config) -> dict: numbered.append(d) return { - "schema": "enodia.response.plan.v1", + "schema": schemas.RESPONSE_PLAN_V1, "plan_id": f"plan-{incident_id}", "incident_id": incident_id, "created_at": datetime.now().astimezone().isoformat(), @@ -242,6 +243,7 @@ def persist_plan(cfg: Config, plan: dict, actor: str = "cli") -> dict[str, str]: audit_path = response_audit_path(cfg) record = { + "schema": schemas.RESPONSE_AUDIT_V1, "time": datetime.now().astimezone().isoformat(), "event": "response_plan_generated", "actor": actor, diff --git a/enodia_sentinel/schemas.py b/enodia_sentinel/schemas.py new file mode 100644 index 0000000..02fb508 --- /dev/null +++ b/enodia_sentinel/schemas.py @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +"""Stable JSON schema identifiers for operator-facing contracts. + +These are lightweight schema IDs, not a full JSON Schema implementation. Tests +pin the minimum v1 fields and types so downstream automation can rely on them +while newer releases remain free to add fields. +""" + +ALERT_V1 = "enodia.alert.v1" +ALERT_SNAPSHOT_V1 = "enodia.alert.snapshot.v1" +INCIDENT_V1 = "enodia.incident.v1" +INCIDENT_VIEW_V1 = "enodia.incident.view.v1" +INCIDENT_BUNDLE_V1 = "enodia.incident.bundle.v1" +STATUS_V1 = "enodia.status.v1" +RESPONSE_PLAN_V1 = "enodia.response.plan.v1" +RESPONSE_AUDIT_V1 = "enodia.response.audit.v1" diff --git a/enodia_sentinel/snapshot.py b/enodia_sentinel/snapshot.py index dab67fb..928f7b9 100644 --- a/enodia_sentinel/snapshot.py +++ b/enodia_sentinel/snapshot.py @@ -15,6 +15,7 @@ import time from datetime import datetime from pathlib import Path +from . import schemas from .alert import Alert, Severity from .config import Config from .system import Process, SystemState @@ -173,6 +174,7 @@ def capture(alerts: list[Alert], state: SystemState, cfg: Config) -> Path: cfg, base.with_suffix(".log").name, alerts, lineage, now.timestamp(), host) report = { + "schema": schemas.ALERT_SNAPSHOT_V1, "time": now.isoformat(), "host": host, "severity": str(severity), diff --git a/enodia_sentinel/web.py b/enodia_sentinel/web.py index 78e8107..c36fd3a 100644 --- a/enodia_sentinel/web.py +++ b/enodia_sentinel/web.py @@ -24,6 +24,7 @@ from pathlib import Path from urllib.parse import parse_qs, unquote, urlparse from . import __version__ +from . import schemas from .config import Config _STATIC = Path(__file__).parent / "static" @@ -199,6 +200,7 @@ def daemon_status(cfg: Config) -> dict: from .selfprotect import heartbeat_age age = heartbeat_age(cfg) return { + "schema": schemas.STATUS_V1, "version": __version__, "running": pid_alive, "total_alerts": len(alerts), @@ -246,7 +248,12 @@ def get_incident(cfg: Config, incident_id: str) -> dict | None: if isinstance(p.get("pid"), int)], }) timeline.sort(key=lambda r: r.get("time", "")) - return {"incident": inc, "timeline": timeline, "snapshots": snapshots} + return { + "schema": schemas.INCIDENT_VIEW_V1, + "incident": inc, + "timeline": timeline, + "snapshots": snapshots, + } def response_plan(cfg: Config, incident_id: str) -> dict | None: diff --git a/tests/test_schema_contracts.py b/tests/test_schema_contracts.py new file mode 100644 index 0000000..5e316e5 --- /dev/null +++ b/tests/test_schema_contracts.py @@ -0,0 +1,222 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +"""Compatibility tests for stable operator-facing JSON contracts.""" +import json +import os +import tempfile +import unittest +from contextlib import redirect_stdout +from io import StringIO +from pathlib import Path +from unittest.mock import patch + +from enodia_sentinel import incident, respond, schemas, snapshot, web +from enodia_sentinel.alert import Alert, Severity +from enodia_sentinel.cli import main +from enodia_sentinel.config import Config + + +def _cfg(tmp: Path) -> Config: + c = Config() + c.log_dir = tmp + return c + + +def _alert() -> Alert: + return Alert( + Severity.CRITICAL, + "reverse_shell", + "rsh:4242", + "pid=4242 peer=[8.8.8.8:4444]", + (4242,), + sid=100010, + classtype="command-and-control", + ) + + +def _assert_types(case: unittest.TestCase, obj: dict, expected: dict): + for key, typ in expected.items(): + case.assertIn(key, obj) + case.assertIsInstance(obj[key], typ, key) + + +class TestSchemaContracts(unittest.TestCase): + def setUp(self): + self.dir = tempfile.TemporaryDirectory() + self.tmp = Path(self.dir.name) + self.cfg = _cfg(self.tmp) + + def tearDown(self): + self.dir.cleanup() + + def _record_incident_with_snapshot(self) -> str: + alert = _alert() + iid = incident.record( + self.cfg, "alert-1.log", [alert], {4242, 100}, 1000.0, "host-a") + self.assertIsNotNone(iid) + (self.tmp / "alert-1.json").write_text(json.dumps({ + "schema": schemas.ALERT_SNAPSHOT_V1, + "time": "2026-06-16T12:00:00-07:00", + "host": "host-a", + "severity": "CRITICAL", + "incident_id": iid, + "alerts": [alert.to_dict()], + "processes": [{"pid": 4242, "comm": "bash"}], + })) + return str(iid) + + def test_alert_v1_contract(self): + alert = _alert().to_dict() + self.assertEqual(schemas.ALERT_V1, "enodia.alert.v1") + _assert_types(self, alert, { + "sid": int, + "severity": str, + "signature": str, + "classtype": str, + "key": str, + "detail": str, + "pids": list, + }) + + def test_alert_snapshot_v1_contract(self): + with patch("enodia_sentinel.snapshot._run", return_value=""): + path = snapshot.capture([_alert()], self._State(), self.cfg) + data = json.loads(path.with_suffix(".json").read_text()) + self.assertEqual(data["schema"], schemas.ALERT_SNAPSHOT_V1) + _assert_types(self, data, { + "schema": str, + "time": str, + "host": str, + "severity": str, + "incident_id": str, + "alerts": list, + "processes": list, + }) + self.assertEqual(data["alerts"][0]["signature"], "reverse_shell") + + def test_incident_v1_contract_and_json_views(self): + iid = self._record_incident_with_snapshot() + inc = incident.load_index(self.cfg)[iid] + self.assertEqual(inc["schema"], schemas.INCIDENT_V1) + _assert_types(self, inc, { + "schema": str, + "id": str, + "host": str, + "first_ts": float, + "last_ts": float, + "first_seen": str, + "last_seen": str, + "severity": str, + "signatures": list, + "sids": list, + "pids": list, + "lineage": list, + "snapshots": list, + "alert_count": int, + }) + + view = web.get_incident(self.cfg, iid) + self.assertEqual(view["schema"], schemas.INCIDENT_VIEW_V1) + _assert_types(self, view, { + "schema": str, + "incident": dict, + "timeline": list, + "snapshots": list, + }) + + os.environ["ENODIA_LOG_DIR"] = str(self.tmp) + try: + buf = StringIO() + with redirect_stdout(buf): + code = main(["incident", "export", iid]) + self.assertEqual(code, 0) + bundle = json.loads(buf.getvalue()) + finally: + os.environ.pop("ENODIA_LOG_DIR", None) + self.assertEqual(bundle["schema"], schemas.INCIDENT_BUNDLE_V1) + _assert_types(self, bundle, { + "schema": str, + "incident": dict, + "snapshots": list, + }) + + def test_old_incident_records_get_schema_backfilled(self): + (self.tmp / "incidents.json").write_text(json.dumps({ + "inc-old": { + "id": "inc-old", + "host": "host-a", + "first_ts": 1.0, + "last_ts": 1.0, + "first_seen": "2026-06-16T12:00:00-07:00", + "last_seen": "2026-06-16T12:00:00-07:00", + "severity": "HIGH", + "signatures": [], + "sids": [], + "pids": [], + "lineage": [], + "snapshots": [], + "alert_count": 0, + }, + })) + inc = incident.load_index(self.cfg)["inc-old"] + self.assertEqual(inc["schema"], schemas.INCIDENT_V1) + + def test_status_v1_contract(self): + status = web.daemon_status(self.cfg) + self.assertEqual(status["schema"], schemas.STATUS_V1) + _assert_types(self, status, { + "schema": str, + "version": str, + "running": bool, + "total_alerts": int, + "counts": dict, + "last_alert": (str, type(None)), + "ebpf": str, + "ebpf_exec": str, + "ebpf_syscall": str, + "host": str, + "heartbeat_age": (float, type(None)), + "heartbeat_stale": bool, + }) + + def test_response_plan_and_audit_contracts(self): + iid = self._record_incident_with_snapshot() + bundle = respond.load_bundle(self.cfg, iid) + plan = respond.build_plan(bundle, self.cfg) + self.assertEqual(plan["schema"], schemas.RESPONSE_PLAN_V1) + _assert_types(self, plan, { + "schema": str, + "plan_id": str, + "incident_id": str, + "created_at": str, + "mode": str, + "apply_supported": bool, + "summary": dict, + "actions": list, + "notes": list, + }) + self.assertEqual(plan["actions"][0]["requires_review"], True) + + artifacts = respond.persist_plan(self.cfg, plan, actor="test") + audit = Path(artifacts["audit_log"]) + record = json.loads(audit.read_text().splitlines()[-1]) + self.assertEqual(record["schema"], schemas.RESPONSE_AUDIT_V1) + _assert_types(self, record, { + "schema": str, + "time": str, + "event": str, + "actor": str, + "plan_id": str, + "incident_id": str, + "mode": str, + "apply_supported": bool, + "action_count": int, + "plan_path": str, + }) + + class _State: + def process(self, _pid): + return None + + +if __name__ == "__main__": + unittest.main()