From 538d1d954a8f717a0c22f364b19c504e4a167181 Mon Sep 17 00:00:00 2001 From: Luna Date: Wed, 22 Jul 2026 01:59:49 -0700 Subject: [PATCH] feat(go): persist Python-compatible incidents --- README.md | 8 +- docs/GO_PORT_HANDOFF.md | 38 ++-- docs/PACKAGING.md | 7 +- docs/ROADMAP.md | 10 +- docs/SURICATA_ASSIMILATION.md | 9 +- go-agent/README.md | 18 +- go-agent/cmd/enodia-sentinel-go/main.go | 5 + go-agent/internal/config/config.go | 32 +++ go-agent/internal/config/config_test.go | 10 +- go-agent/internal/incident/store.go | 256 +++++++++++++++++++++++ go-agent/internal/incident/store_test.go | 89 ++++++++ go-agent/internal/snapshot/store.go | 64 +++++- go-agent/internal/snapshot/store_test.go | 55 +++++ 13 files changed, 555 insertions(+), 46 deletions(-) create mode 100644 go-agent/internal/incident/store.go create mode 100644 go-agent/internal/incident/store_test.go diff --git a/README.md b/README.md index 87f088b..e09918a 100644 --- a/README.md +++ b/README.md @@ -72,9 +72,11 @@ hardened systemd unit writes JSONL to journald, retains a bounded copy of the event stream plus baseline state and an atomic heartbeat under `/var/lib/enodia-sentinel-go`, and fails open to polling if a probe cannot load. It also writes bounded `enodia.alert.snapshot.v1` JSON/text pairs with truthful -status counts. It does not replace, install over, or write state for the Python -daemon; Python remains authoritative for richer forensic enrichment, incidents, -assurance, notifications, and the remaining subsystem parity. +status counts and a private Python-compatible `enodia.incident.v1` index using +process-lineage/time-window grouping. It does not replace, install over, or +write state for the Python daemon; Python remains authoritative for richer +forensic enrichment, assurance, notifications, management consumers, and the +remaining subsystem parity. ## Why these detectors diff --git a/docs/GO_PORT_HANDOFF.md b/docs/GO_PORT_HANDOFF.md index 810a4fe..4cf7c81 100644 --- a/docs/GO_PORT_HANDOFF.md +++ b/docs/GO_PORT_HANDOFF.md @@ -1,8 +1,8 @@ # Go Port Handoff -Saved: 2026-07-22T01:40:52-07:00 +Saved: 2026-07-22T01:58:38-07:00 Branch: `main` -Base commit: `6a06eba` (`docs(behavior): add behavioral spec for sweep + baseline lifecycle`) +Base commit: `f85c2e8` (`feat(go): advance validation sidecar toward production`) Status: implemented and green, but uncommitted ## Worktree warning @@ -10,13 +10,10 @@ Status: implemented and green, but uncommitted The checkout is intentionally dirty and contains work from multiple related continuations. Do not reset, clean, or broadly restage it. -- At this checkpoint, `git status --short` has 62 entries (105 when every - untracked file is expanded instead of collapsing directories). -- This handoff is itself currently untracked and should be included when the - Go migration tranche is eventually committed. -- `docs/behavior/02-poll-detectors.md` and `docs/behavior/index.md` already had - staged content plus later unstaged edits (`AM` / `MM`). This save did not - change the index. +- The completed validation-sidecar tranche was committed as `f85c2e8` before + this continuation. The current Go incident tranche is not yet committed. +- At this checkpoint, `git status --short` has 20 entries with untracked + directories collapsed. - The Python GUI files and tests are separate pre-existing work. Preserve them while continuing the Go port. - `build/` is ignored and contains only local build output. @@ -84,9 +81,18 @@ static binary. - `enodia.status.v1` totals, severity counts, and last-alert time are populated only from parseable retained Go snapshots. Python's unchanged `list_alerts` and `get_alert` data layer successfully consumed a Go-produced pair. -- Snapshot incident IDs remain null and enrichment currently contains only a - Go-source marker plus process detail; incident indexing, rich enrichment, - assurance chaining, and notification fan-out are not yet ported. +- Snapshot incident IDs now reference an atomic private `incidents.json` using + the required `enodia.incident.v1` fields. Grouping mirrors Python's process- + lineage-first, time-window fallback behavior, the index is bounded to 1000 + recent incidents, and same-second alert batches update one incident without + duplicating the snapshot name. +- Incident severity/signatures/SIDs/PIDs/lineage/timestamps are updated + additively and correlation SID `100080` is persisted. Python's unchanged + `list_incidents` and `get_incident` consumers successfully loaded a live + Go-produced index, snapshot, and timeline. +- Enrichment currently contains only a Go-source marker plus process detail; + rich enrichment, assurance chaining, and notification fan-out are not yet + ported. - No live system service was installed or enabled during development. ## Last green verification @@ -131,14 +137,12 @@ the suite still exits successfully. ## Resume here -1. Add Python-compatible incident grouping/index persistence so retained Go - snapshots can carry real `incident_id` values and correlation timelines. -2. Port bounded post-alert enrichment and local assurance chaining without +1. Port bounded post-alert enrichment and local assurance chaining without adding destructive response behavior or blocking the event loop on slow collectors. -3. Connect the isolated Go snapshot/event state to explicit read-only +2. Connect the isolated Go snapshot/event state to explicit read-only management consumers while keeping Python authoritative. -4. Continue broader Phase 3 rule metadata/state parity before considering any +3. Continue broader Phase 3 rule metadata/state parity before considering any default-service or package cutover. Keep the validation sidecar opt-in, preserve its separate state tree, and keep diff --git a/docs/PACKAGING.md b/docs/PACKAGING.md index 8b475d3..b13c887 100644 --- a/docs/PACKAGING.md +++ b/docs/PACKAGING.md @@ -91,8 +91,11 @@ Alert events are additionally retained as private Same-second events are merged into one snapshot, required `enodia.alert.snapshot.v1` fields are preserved, and the shared `max_snapshots` / `max_snapshot_age_days` settings bound retention. The current -Go enrichment block is intentionally minimal; Python remains authoritative for -incident grouping, rich enrichment, assurance chaining, and notifications. +sidecar also writes a private atomic `incidents.json` with the required +`enodia.incident.v1` fields, process-lineage/time-window grouping, and additive +correlation evidence. The Go enrichment block is intentionally minimal; Python +remains authoritative for rich enrichment, assurance chaining, notifications, +and management consumers. Each successful sweep atomically refreshes `/var/lib/enodia-sentinel-go/heartbeat` as a Unix timestamp with mode `0600`; diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 2a31b27..85afd77 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -288,10 +288,12 @@ Exit criteria: journal JSONL. A bounded rotating JSONL copy provides isolated retained event history, and an atomic heartbeat gives external watchdogs a non-journal liveness signal. Bounded Python-schema-compatible JSON/text alert snapshots - now provide truthful retained counts and basic process context. The Python - service remains the production default until incident grouping, rich - enrichment/assurance, management-console consumers, and broader subsystem - parity land. + now provide truthful retained counts and basic process context. A private, + atomic `enodia.incident.v1` index now applies the shared lineage-first/time- + window grouping, assigns real snapshot `incident_id` values, and persists + correlation SID `100080`. The Python service remains the production default + until rich enrichment/assurance, management-console consumers, and broader + subsystem parity land. ## Backlog diff --git a/docs/SURICATA_ASSIMILATION.md b/docs/SURICATA_ASSIMILATION.md index 9a5c95a..78757da 100644 --- a/docs/SURICATA_ASSIMILATION.md +++ b/docs/SURICATA_ASSIMILATION.md @@ -249,10 +249,11 @@ signature keeps `sid` / `signature` / `classtype` / tests / docs. isolated baseline state, journal JSONL, a bounded retained JSONL stream, and an atomic health marker. All current typed-host families now have native transports. Bounded `enodia.alert.snapshot.v1` JSON/text pairs now retain - alerts with basic process context and feed truthful status counts. Incident - grouping, rich enrichment/assurance, management-consumer integration, and - broader parity must land before the bcc/Python runtime dependency can be - dropped. + alerts with basic process context and feed truthful status counts. An atomic + `enodia.incident.v1` index supplies lineage/time grouping, snapshot IDs, and + correlation evidence. Rich enrichment/assurance, management-consumer + integration, and broader parity must land before the bcc/Python runtime + dependency can be dropped. - **Phase 3 — Rule-engine parity + assimilation.** `rev`/`reference`/`metadata`, statebits (flowbits), thresholding, suppression, and — if rule count warrants — the MPM prefilter. `rules list/show/test/docs` parity. diff --git a/go-agent/README.md b/go-agent/README.md index a2c6a58..5b7b05f 100644 --- a/go-agent/README.md +++ b/go-agent/README.md @@ -46,11 +46,14 @@ Current scope: - optional `enodia.alert.snapshot.v1` JSON/text pairs (`--snapshot-dir`) with same-second alert batching, best-effort `/proc` context, and Python-compatible count/age pruning; +- Python-compatible `enodia.incident.v1` grouping and atomic `incidents.json` + persistence, using process lineage first and the shared time-window fallback, + including additive SID `100080` correlation evidence; - a shared fixture checked against the Python detector by `scripts/check-go-parity.py`. -It does not yet provide Python's incident grouping, rich enrichment, assurance -chain, or notification fan-out for those snapshots, and it does not replace +It does not yet provide Python's rich enrichment, assurance chain, notification +fan-out, or management consumers for those snapshots, and it does not replace `enodia-sentinel`. The module requires Go 1.25 or newer; development is currently verified with Go 1.26.5. Run a single terminal-visible stateless sweep: @@ -105,10 +108,13 @@ Alert envelopes are also batched into private `alert-YYYYMMDD-HHMMSS.{json,log}` pairs under the same isolated directory. The JSON side implements the required `enodia.alert.snapshot.v1` fields and the service applies shared `max_snapshots` / `max_snapshot_age_days` limits. Status -counts are populated only from parseable retained snapshots. Enrichment is -currently limited to a Go-source marker plus best-effort process detail; -incident indexing and the Python forensic enrichment/assurance pipeline remain -future parity work. +counts are populated only from parseable retained snapshots. Each snapshot is +linked to an incident in a private atomic `incidents.json` index. Grouping uses +the configured `incident_tracking`, `incident_window`, and +`incident_lineage_depth` settings; same-second alert batches update one incident +and one snapshot name idempotently. Enrichment is currently limited to a +Go-source marker plus best-effort process detail; the Python forensic +enrichment/assurance and notification pipelines remain future parity work. After every successfully emitted status record, the service atomically updates `/var/lib/enodia-sentinel-go/heartbeat` with the current Unix timestamp (mode diff --git a/go-agent/cmd/enodia-sentinel-go/main.go b/go-agent/cmd/enodia-sentinel-go/main.go index 65ba054..cee64df 100644 --- a/go-agent/cmd/enodia-sentinel-go/main.go +++ b/go-agent/cmd/enodia-sentinel-go/main.go @@ -241,6 +241,11 @@ func run() error { if err != nil { return err } + if err := snapshotStore.ConfigureIncidents( + cfg.IncidentTracking, cfg.IncidentWindow, cfg.IncidentLineageDepth, + ); err != nil { + return err + } } notifier := sdnotify.FromEnvironment() if err := notifier.Notify("READY=1\nSTATUS=initialization complete; poll loop ready"); err != nil { diff --git a/go-agent/internal/config/config.go b/go-agent/internal/config/config.go index 56ffefa..6b97953 100644 --- a/go-agent/internal/config/config.go +++ b/go-agent/internal/config/config.go @@ -61,6 +61,9 @@ type Config struct { HeartbeatMaxAge int MaxSnapshots int MaxSnapshotAgeDays int + IncidentTracking bool + IncidentWindow int + IncidentLineageDepth int Detectors map[string]bool InputSnooperAllowComms map[string]bool CredentialAccessAllowComms map[string]bool @@ -97,6 +100,9 @@ func Default() Config { HeartbeatMaxAge: 120, MaxSnapshots: 300, MaxSnapshotAgeDays: 60, + IncidentTracking: true, + IncidentWindow: 1800, + IncidentLineageDepth: 8, Detectors: detectors, InputSnooperAllowComms: stringSet(defaultInputSnooperAllowComms), CredentialAccessAllowComms: stringSet(defaultCredentialAccessAllowComms), @@ -204,6 +210,24 @@ func Load(path string) (Config, error) { return Config{}, fmt.Errorf("max_snapshot_age_days must be non-negative: %q", value) } cfg.MaxSnapshotAgeDays = days + case "incident_tracking": + enabled, err := boolean(value) + if err != nil { + return Config{}, fmt.Errorf("incident_tracking: %w", err) + } + cfg.IncidentTracking = enabled + case "incident_window": + window, err := nonNegativeInt(value) + if err != nil { + return Config{}, fmt.Errorf("incident_window: %w", err) + } + cfg.IncidentWindow = window + case "incident_lineage_depth": + depth, err := nonNegativeInt(value) + if err != nil { + return Config{}, fmt.Errorf("incident_lineage_depth: %w", err) + } + cfg.IncidentLineageDepth = depth case "detectors": names, err := stringArray(value) if err != nil { @@ -351,6 +375,14 @@ func nonNegativeSeconds(value string) (time.Duration, error) { return time.Duration(seconds * float64(time.Second)), nil } +func nonNegativeInt(value string) (int, error) { + parsed, err := strconv.Atoi(value) + if err != nil || parsed < 0 { + return 0, fmt.Errorf("must be non-negative: %q", value) + } + return parsed, nil +} + func boolean(value string) (bool, error) { switch value { case "true": diff --git a/go-agent/internal/config/config_test.go b/go-agent/internal/config/config_test.go index 73a16db..e842bf4 100644 --- a/go-agent/internal/config/config_test.go +++ b/go-agent/internal/config/config_test.go @@ -15,7 +15,9 @@ func TestDefaultsMatchPython(t *testing.T) { cfg.BaselineGrace != 10*time.Second || cfg.SUIDRefresh != time.Hour || cfg.SUIDScanInterval != time.Minute || cfg.HeartbeatMaxAge != 120 || cfg.MaxSnapshots != 300 || - cfg.MaxSnapshotAgeDays != 60 || !cfg.FirstSeenEnabled { + cfg.MaxSnapshotAgeDays != 60 || !cfg.IncidentTracking || + cfg.IncidentWindow != 1800 || cfg.IncidentLineageDepth != 8 || + !cfg.FirstSeenEnabled { t.Fatalf("unexpected defaults: %+v", cfg) } if !cfg.Enabled("deleted_exe") { @@ -34,6 +36,9 @@ suid_scan_interval = 15 heartbeat_max_age = 45 max_snapshots = 25 max_snapshot_age_days = 7 +incident_tracking = false +incident_window = 90 +incident_lineage_depth = 3 detectors = [ "deleted_exe", "egress", @@ -67,7 +72,8 @@ unknown_future_key = true if cfg.SampleInterval != 1500*time.Millisecond || cfg.Cooldown != 30*time.Second || cfg.BaselineGrace != 2500*time.Millisecond || cfg.SUIDRefresh != 2*time.Minute || cfg.SUIDScanInterval != 15*time.Second || - cfg.HeartbeatMaxAge != 45 || cfg.MaxSnapshots != 25 || cfg.MaxSnapshotAgeDays != 7 { + cfg.HeartbeatMaxAge != 45 || cfg.MaxSnapshots != 25 || cfg.MaxSnapshotAgeDays != 7 || + cfg.IncidentTracking || cfg.IncidentWindow != 90 || cfg.IncidentLineageDepth != 3 { t.Fatalf("unexpected parsed config: %+v", cfg) } if !cfg.Enabled("deleted_exe") || !cfg.Enabled("egress") || cfg.Enabled("reverse_shell") { diff --git a/go-agent/internal/incident/store.go b/go-agent/internal/incident/store.go new file mode 100644 index 0000000..ab3dfec --- /dev/null +++ b/go-agent/internal/incident/store.go @@ -0,0 +1,256 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +// Package incident persists Python-compatible incident grouping for retained +// Go alert snapshots. Grouping is process-lineage first and time-window second; +// it adds evidence without hiding or replacing the raw alerts. +package incident + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "sync" + "time" + + "codeberg.org/anassaeneroi/enodia-sentinal/go-agent/internal/correlation" + "codeberg.org/anassaeneroi/enodia-sentinal/go-agent/internal/model" +) + +const ( + Schema = "enodia.incident.v1" + IndexName = "incidents.json" + MaxIncidents = 1000 +) + +type Record struct { + Schema string `json:"schema"` + ID string `json:"id"` + Host string `json:"host"` + FirstTS float64 `json:"first_ts"` + LastTS float64 `json:"last_ts"` + FirstSeen string `json:"first_seen"` + LastSeen string `json:"last_seen"` + Severity string `json:"severity"` + Signatures []string `json:"signatures"` + SIDs []int `json:"sids"` + PIDs []int `json:"pids"` + Lineage []int `json:"lineage"` + Snapshots []string `json:"snapshots"` + AlertCount int `json:"alert_count"` + Correlations []correlation.Record `json:"correlations"` +} + +type Store struct { + Dir string + Window int + Depth int + Enabled bool + + mu sync.Mutex +} + +// New validates incident settings and preflights the isolated state directory. +func New(dir string, enabled bool, window, depth int) (*Store, error) { + if dir == "" { + return nil, fmt.Errorf("incident directory is required") + } + if window < 0 || depth < 0 { + return nil, fmt.Errorf("incident window and lineage depth must be non-negative") + } + if err := os.MkdirAll(dir, 0o750); err != nil { + return nil, fmt.Errorf("create incident directory: %w", err) + } + return &Store{Dir: dir, Window: window, Depth: depth, Enabled: enabled}, nil +} + +// RecordAlert attaches one newly appended alert to a snapshot's incident. +// preferredID keeps all alerts merged into the same snapshot in one incident. +// The snapshot name itself is appended idempotently. +func (s *Store) RecordAlert(snapshotName string, alert model.Alert, lineage map[int]bool, + when time.Time, host, preferredID string) (string, error) { + if !s.Enabled { + return "", nil + } + s.mu.Lock() + defer s.mu.Unlock() + + index, err := load(filepath.Join(s.Dir, IndexName)) + if err != nil { + return "", err + } + id := "" + if preferredID != "" && index[preferredID] != nil { + id = preferredID + } + if id == "" { + candidates := make([]correlation.IndexIncident, 0, len(index)) + for _, item := range index { + candidates = append(candidates, correlation.IndexIncident{ + ID: item.ID, Host: item.Host, LastTimestamp: item.LastTS, Lineage: item.Lineage, + }) + } + id = correlation.Assign(candidates, lineage, float64(when.UnixNano())/1e9, host, s.Window) + } + whenFloat := float64(when.UnixNano()) / 1e9 + whenISO := when.Format(time.RFC3339Nano) + if id == "" { + id, err = newID(when) + if err != nil { + return "", err + } + index[id] = &Record{ + Schema: Schema, ID: id, Host: host, FirstTS: whenFloat, LastTS: whenFloat, + FirstSeen: whenISO, LastSeen: whenISO, Severity: alert.Severity, + Signatures: []string{}, SIDs: []int{}, PIDs: []int{}, Lineage: []int{}, + Snapshots: []string{}, Correlations: []correlation.Record{}, + } + } + item := index[id] + item.Schema = Schema + item.LastTS = whenFloat + item.LastSeen = whenISO + item.Severity = correlation.MaxSeverity(item.Severity, alert.Severity) + item.Signatures = appendUniqueString(item.Signatures, alert.Signature) + if alert.SID != 0 { + item.SIDs = appendUniqueInt(item.SIDs, alert.SID) + } + item.PIDs = sortedUnion(item.PIDs, alert.PIDs) + item.Lineage = sortedUnion(item.Lineage, mapKeys(lineage)) + item.Snapshots = appendUniqueString(item.Snapshots, snapshotName) + item.AlertCount++ + item.Correlations = correlation.Correlate(correlation.Incident{ + FirstTimestamp: item.FirstTS, LastTimestamp: item.LastTS, Signatures: item.Signatures, + }, nil) + for _, match := range item.Correlations { + item.SIDs = appendUniqueInt(item.SIDs, match.SID) + item.Severity = correlation.MaxSeverity(item.Severity, match.Severity) + } + if err := save(filepath.Join(s.Dir, IndexName), index); err != nil { + return "", err + } + return id, nil +} + +// LoadIndex reads the durable index for compatibility tests and consumers. +func LoadIndex(dir string) (map[string]*Record, error) { + return load(filepath.Join(dir, IndexName)) +} + +func load(path string) (map[string]*Record, error) { + raw, err := os.ReadFile(path) + if os.IsNotExist(err) { + return map[string]*Record{}, nil + } + if err != nil { + return nil, fmt.Errorf("read incident index: %w", err) + } + index := map[string]*Record{} + if json.Unmarshal(raw, &index) != nil { + // Python treats malformed legacy state as an empty index. + return map[string]*Record{}, nil + } + for id, item := range index { + if item == nil { + delete(index, id) + continue + } + if item.Schema == "" { + item.Schema = Schema + } + if item.Correlations == nil { + item.Correlations = []correlation.Record{} + } + } + return index, nil +} + +func save(path string, index map[string]*Record) error { + if len(index) > MaxIncidents { + items := make([]*Record, 0, len(index)) + for _, item := range index { + items = append(items, item) + } + sort.Slice(items, func(i, j int) bool { + if items[i].LastTS == items[j].LastTS { + return items[i].ID < items[j].ID + } + return items[i].LastTS > items[j].LastTS + }) + trimmed := make(map[string]*Record, MaxIncidents) + for _, item := range items[:MaxIncidents] { + trimmed[item.ID] = item + } + index = trimmed + } + raw, err := json.MarshalIndent(index, "", " ") + if err != nil { + return fmt.Errorf("encode incident index: %w", err) + } + temporary := path + ".tmp" + if err := os.WriteFile(temporary, append(raw, '\n'), 0o600); err != nil { + return fmt.Errorf("write incident index: %w", err) + } + if err := os.Rename(temporary, path); err != nil { + return fmt.Errorf("replace incident index: %w", err) + } + if err := os.Chmod(path, 0o600); err != nil { + return fmt.Errorf("protect incident index: %w", err) + } + return nil +} + +func newID(when time.Time) (string, error) { + suffix := make([]byte, 2) + if _, err := rand.Read(suffix); err != nil { + return "", fmt.Errorf("generate incident id: %w", err) + } + return "inc-" + when.Local().Format("20060102-150405") + "-" + hex.EncodeToString(suffix), nil +} + +func appendUniqueString(values []string, value string) []string { + for _, current := range values { + if current == value { + return values + } + } + return append(values, value) +} + +func appendUniqueInt(values []int, value int) []int { + for _, current := range values { + if current == value { + return values + } + } + return append(values, value) +} + +func sortedUnion(left, right []int) []int { + values := make(map[int]bool, len(left)+len(right)) + for _, value := range left { + values[value] = true + } + for _, value := range right { + values[value] = true + } + result := make([]int, 0, len(values)) + for value := range values { + result = append(result, value) + } + sort.Ints(result) + return result +} + +func mapKeys(values map[int]bool) []int { + result := make([]int, 0, len(values)) + for value, enabled := range values { + if enabled { + result = append(result, value) + } + } + return result +} diff --git a/go-agent/internal/incident/store_test.go b/go-agent/internal/incident/store_test.go new file mode 100644 index 0000000..1f22a37 --- /dev/null +++ b/go-agent/internal/incident/store_test.go @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package incident + +import ( + "os" + "path/filepath" + "testing" + "time" + + "codeberg.org/anassaeneroi/enodia-sentinal/go-agent/internal/correlation" + "codeberg.org/anassaeneroi/enodia-sentinal/go-agent/internal/model" +) + +func TestRecordAlertGroupsLineageAndPersistsCorrelation(t *testing.T) { + directory := t.TempDir() + store, err := New(directory, true, 1800, 8) + if err != nil { + t.Fatal(err) + } + firstTime := time.Unix(1000, 0).UTC() + first, err := store.RecordAlert("alert-1.log", model.Alert{ + SID: 100003, Severity: "HIGH", Signature: "exec_rule.web-rce", PIDs: []int{4242}, + }, map[int]bool{4242: true, 999: true}, firstTime, "host-a", "") + if err != nil { + t.Fatal(err) + } + second, err := store.RecordAlert("alert-2.log", model.Alert{ + SID: 100067, Severity: "HIGH", Signature: "host_rule.suspicious-egress", PIDs: []int{4243}, + }, map[int]bool{4243: true, 999: true}, firstTime.Add(50*time.Second), "host-a", "") + if err != nil { + t.Fatal(err) + } + if first == "" || second != first { + t.Fatalf("incident ids first=%q second=%q", first, second) + } + index, err := LoadIndex(directory) + if err != nil { + t.Fatal(err) + } + item := index[first] + if len(index) != 1 || item == nil || item.Schema != Schema || item.AlertCount != 2 || + item.Severity != "CRITICAL" || len(item.Snapshots) != 2 || + len(item.Correlations) != 1 || item.Correlations[0].SID != correlation.SIDMultiStageIntrusion { + t.Fatalf("index=%#v", index) + } + if info, err := os.Stat(filepath.Join(directory, IndexName)); err != nil || info.Mode().Perm() != 0o600 { + t.Fatalf("incident index mode info=%v err=%v", info, err) + } +} + +func TestPreferredIncidentMakesSnapshotAppendIdempotent(t *testing.T) { + store, err := New(t.TempDir(), true, 1800, 8) + if err != nil { + t.Fatal(err) + } + when := time.Unix(1000, 0).UTC() + id, err := store.RecordAlert("alert-1.log", model.Alert{ + SID: 1, Severity: "HIGH", Signature: "one", PIDs: []int{10}, + }, map[int]bool{10: true}, when, "host-a", "") + if err != nil { + t.Fatal(err) + } + if _, err := store.RecordAlert("alert-1.log", model.Alert{ + SID: 2, Severity: "CRITICAL", Signature: "two", PIDs: []int{20}, + }, map[int]bool{20: true}, when.Add(500*time.Millisecond), "host-a", id); err != nil { + t.Fatal(err) + } + index, _ := LoadIndex(store.Dir) + item := index[id] + if len(item.Snapshots) != 1 || item.AlertCount != 2 || len(item.PIDs) != 2 || item.Severity != "CRITICAL" { + t.Fatalf("incident=%#v", item) + } +} + +func TestDisabledStoreDoesNotWriteIndex(t *testing.T) { + directory := t.TempDir() + store, err := New(directory, false, 1800, 8) + if err != nil { + t.Fatal(err) + } + id, err := store.RecordAlert("alert.log", model.Alert{SID: 1, Severity: "HIGH", Signature: "test"}, nil, time.Now(), "host-a", "") + if err != nil || id != "" { + t.Fatalf("id=%q err=%v", id, err) + } + if _, err := os.Stat(filepath.Join(directory, IndexName)); !os.IsNotExist(err) { + t.Fatalf("disabled index exists: %v", err) + } +} diff --git a/go-agent/internal/snapshot/store.go b/go-agent/internal/snapshot/store.go index 6efd272..ce58192 100644 --- a/go-agent/internal/snapshot/store.go +++ b/go-agent/internal/snapshot/store.go @@ -1,9 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Package snapshot retains Python-compatible alert snapshot pairs for the Go -// migration service. It intentionally leaves incident grouping and rich -// post-alert enrichment to later parity tranches while preserving the stable -// v1 fields consumed by existing read-only tools. +// migration service, including process-lineage incident linkage. Rich post- +// alert enrichment remains a later parity tranche. package snapshot import ( @@ -17,6 +16,8 @@ import ( "sync" "time" + "codeberg.org/anassaeneroi/enodia-sentinal/go-agent/internal/correlation" + "codeberg.org/anassaeneroi/enodia-sentinal/go-agent/internal/incident" "codeberg.org/anassaeneroi/enodia-sentinal/go-agent/internal/model" ) @@ -59,6 +60,8 @@ type Store struct { MaxAgeDays int Now func() time.Time Collect func(int) ProcessDetail + ParentOf func(int) int + Incidents *incident.Store mu sync.Mutex } @@ -96,12 +99,24 @@ func New(dir, procRoot string, maxCount, maxAgeDays int) (*Store, error) { } store := &Store{Dir: dir, ProcRoot: procRoot, MaxCount: maxCount, MaxAgeDays: maxAgeDays, Now: time.Now} store.Collect = func(pid int) ProcessDetail { return collectProcess(procRoot, pid) } + store.ParentOf = func(pid int) int { return readParent(procRoot, pid) } if err := store.Prune(); err != nil { return nil, err } return store, nil } +// ConfigureIncidents preflights Python-compatible incident persistence before +// service readiness. A disabled tracker leaves incident_id null. +func (s *Store) ConfigureIncidents(enabled bool, window, lineageDepth int) error { + tracker, err := incident.New(s.Dir, enabled, window, lineageDepth) + if err != nil { + return err + } + s.Incidents = tracker + return nil +} + // CaptureEvent merges one alert envelope into its second-granularity snapshot. // Python filenames use second precision, so merging avoids overwriting sibling // alerts emitted by one poll sweep or concurrent kernel sources. @@ -145,6 +160,19 @@ func (s *Store) CaptureEvent(event map[string]any) (string, error) { report.Alerts = append(report.Alerts, alert) report.Severity = maxSeverity(report.Severity, alert.Severity) report.Processes = s.collectProcesses(report.Alerts) + if s.Incidents != nil { + pids := alertPIDs(report.Alerts) + lineage := correlation.LineageOf(pids, s.ParentOf, s.Incidents.Depth) + preferred := "" + if report.IncidentID != nil { + preferred = *report.IncidentID + } + // Incident persistence is best-effort, matching Python: an index problem + // must never discard the forensic snapshot itself. + if id, err := s.Incidents.RecordAlert(filepath.Base(base+".log"), alert, lineage, captured, host, preferred); err == nil && id != "" { + report.IncidentID = &id + } + } if report.Enrichment == nil { report.Enrichment = map[string]any{"source": "enodia-sentinel-go"} } @@ -161,6 +189,15 @@ func (s *Store) CaptureEvent(event map[string]any) (string, error) { } func (s *Store) collectProcesses(alerts []model.Alert) []ProcessDetail { + pids := alertPIDs(alerts) + result := make([]ProcessDetail, 0, len(pids)) + for _, pid := range pids { + result = append(result, s.Collect(pid)) + } + return result +} + +func alertPIDs(alerts []model.Alert) []int { seen := map[int]bool{} var pids []int for _, alert := range alerts { @@ -172,11 +209,7 @@ func (s *Store) collectProcesses(alerts []model.Alert) []ProcessDetail { } } sort.Ints(pids) - result := make([]ProcessDetail, 0, len(pids)) - for _, pid := range pids { - result = append(result, s.Collect(pid)) - } - return result + return pids } // SnapshotStats prunes first, then derives status claims from retained, @@ -312,6 +345,9 @@ func formatText(report Report) string { fmt.Fprintln(&output, "Time: ", report.Time) fmt.Fprintln(&output, "Host: ", report.Host) fmt.Fprintln(&output, "Severity:", report.Severity) + if report.IncidentID != nil { + fmt.Fprintln(&output, "Incident:", *report.IncidentID) + } fmt.Fprintln(&output, "\n## Triggering detections") for _, alert := range report.Alerts { fmt.Fprintf(&output, " [%s] sid:%d %s (%s) — %s\n", alert.Severity, alert.SID, alert.Signature, alert.Classtype, alert.Detail) @@ -379,6 +415,18 @@ func readRaw(path string) string { func readText(path string) string { return strings.TrimSpace(readRaw(path)) } +func readParent(procRoot string, pid int) int { + status := readRaw(filepath.Join(procRoot, strconv.Itoa(pid), "status")) + for _, line := range strings.Split(status, "\n") { + fields := strings.Fields(line) + if len(fields) >= 2 && fields[0] == "PPid:" { + parent, _ := strconv.Atoi(fields[1]) + return parent + } + } + return 0 +} + func readLink(path string) string { target, _ := os.Readlink(path) return target diff --git a/go-agent/internal/snapshot/store_test.go b/go-agent/internal/snapshot/store_test.go index 797a0ab..9baa139 100644 --- a/go-agent/internal/snapshot/store_test.go +++ b/go-agent/internal/snapshot/store_test.go @@ -6,6 +6,7 @@ import ( "encoding/json" "os" "path/filepath" + "strings" "testing" "time" @@ -69,6 +70,60 @@ func TestCaptureEventWritesCompatiblePairAndMergesSameSecond(t *testing.T) { } } +func TestCaptureEventLinksMergedSnapshotToOneIncident(t *testing.T) { + store, err := New(t.TempDir(), "/unused", 10, 30) + if err != nil { + t.Fatal(err) + } + if err := store.ConfigureIncidents(true, 1800, 8); err != nil { + t.Fatal(err) + } + parents := map[int]int{42: 7, 7: 1, 99: 1} + store.ParentOf = func(pid int) int { return parents[pid] } + store.Collect = func(pid int) ProcessDetail { + return ProcessDetail{PID: pid, PPID: parents[pid], FDs: map[string]string{}} + } + first := alertEvent("2026-07-22T10:11:12.100-07:00", model.Alert{ + SID: 100003, Severity: "HIGH", Signature: "exec_rule.web-rce", Classtype: "execution", Key: "one", Detail: "first", PIDs: []int{42}, + }) + second := alertEvent("2026-07-22T10:11:12.900-07:00", model.Alert{ + SID: 100067, Severity: "HIGH", Signature: "host_rule.suspicious-egress", Classtype: "c2", Key: "two", Detail: "second", PIDs: []int{99}, + }) + if _, err := store.CaptureEvent(first); err != nil { + t.Fatal(err) + } + if _, err := store.CaptureEvent(second); err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(filepath.Join(store.Dir, "alert-20260722-101112.json")) + if err != nil { + t.Fatal(err) + } + var report Report + if err := json.Unmarshal(raw, &report); err != nil { + t.Fatal(err) + } + if report.IncidentID == nil || *report.IncidentID == "" { + t.Fatalf("missing incident id: %#v", report) + } + indexRaw, err := os.ReadFile(filepath.Join(store.Dir, "incidents.json")) + if err != nil { + t.Fatal(err) + } + var index map[string]map[string]any + if err := json.Unmarshal(indexRaw, &index); err != nil { + t.Fatal(err) + } + item := index[*report.IncidentID] + if item["schema"] != "enodia.incident.v1" || item["alert_count"] != float64(2) || len(item["snapshots"].([]any)) != 1 { + t.Fatalf("incident=%#v", item) + } + textRaw, err := os.ReadFile(filepath.Join(store.Dir, "alert-20260722-101112.log")) + if err != nil || !strings.Contains(string(textRaw), "Incident: "+*report.IncidentID) { + t.Fatalf("text incident missing err=%v text=%q", err, textRaw) + } +} + func TestRetentionPrunesCountAndAgeAsPairs(t *testing.T) { store, err := New(t.TempDir(), "/unused", 2, 1) if err != nil {