feat(go): enrich snapshots asynchronously

This commit is contained in:
Luna 2026-07-22 03:10:26 -07:00
parent 1ab4add205
commit eff31b3a82
No known key found for this signature in database
7 changed files with 270 additions and 14 deletions

View file

@ -67,9 +67,18 @@ type Store struct {
LineageDepth int
Assurance *assurance.Chain
slowJobs chan slowJob
slowClosed bool
slowWG sync.WaitGroup
mu sync.Mutex
}
type slowJob struct {
jsonPath string
logPath string
}
// New preflights the isolated snapshot directory before service readiness.
func New(dir, procRoot string, maxCount, maxAgeDays int) (*Store, error) {
if dir == "" {
@ -126,6 +135,36 @@ func (s *Store) ConfigureIncidents(enabled bool, window, lineageDepth int) error
return nil
}
// EnableSlowEnrichment starts one bounded worker. It deliberately never runs
// package-manager commands: queue saturation or collector failure only leaves
// fields unknown and can never delay or reject an alert event.
func (s *Store) EnableSlowEnrichment() {
s.mu.Lock()
defer s.mu.Unlock()
if s.slowJobs != nil || s.slowClosed {
return
}
s.slowJobs = make(chan slowJob, 16)
go s.runSlowEnrichment(s.slowJobs)
}
// Close drains accepted slow-enrichment work during a clean service shutdown.
func (s *Store) Close() {
s.mu.Lock()
if s.slowJobs == nil || s.slowClosed {
s.mu.Unlock()
return
}
close(s.slowJobs)
s.slowClosed = true
s.mu.Unlock()
s.slowWG.Wait()
}
// FlushSlow waits for work accepted before the call. It is used by clean
// shutdown/tests; alert emission never waits for this collector.
func (s *Store) FlushSlow() { s.slowWG.Wait() }
// CaptureEvent merges one alert envelope into its second-granularity snapshot.
// Python filenames use second precision, so merging avoids overwriting sibling
// alerts emitted by one poll sweep or concurrent kernel sources.
@ -200,9 +239,112 @@ func (s *Store) CaptureEvent(event map[string]any) (string, error) {
if err := s.pruneLocked(); err != nil {
return "", err
}
s.enqueueSlow(slowJob{jsonPath: jsonPath, logPath: base + ".log"})
return filepath.Base(base + ".log"), nil
}
func (s *Store) enqueueSlow(job slowJob) {
if s.slowJobs == nil || s.slowClosed {
return
}
s.slowWG.Add(1)
select {
case s.slowJobs <- job:
default:
s.slowWG.Done()
}
}
func (s *Store) runSlowEnrichment(jobs <-chan slowJob) {
for job := range jobs {
s.applySlowEnrichment(job)
s.slowWG.Done()
}
}
func (s *Store) applySlowEnrichment(job slowJob) {
raw, err := os.ReadFile(job.jsonPath)
if err != nil {
return
}
var report Report
if json.Unmarshal(raw, &report) != nil || report.Schema != Schema {
return
}
processes := enrichmentProcesses(report.Processes)
paths := enrichmentPaths(report.Enrichment)
updates := enrich.CollectSlow(processes, paths)
// All slow I/O has completed. Take the lock only to merge into the newest
// same-second report, so a concurrent alert capture never loses evidence.
s.mu.Lock()
defer s.mu.Unlock()
raw, err = os.ReadFile(job.jsonPath)
if err != nil || json.Unmarshal(raw, &report) != nil || report.Schema != Schema {
return
}
mergeSlow(&report, updates)
if err := writeJSONAtomic(job.jsonPath, report); err != nil {
return
}
if err := writeAtomic(job.logPath, []byte(formatText(report))); err != nil {
return
}
if s.Assurance != nil {
captured, err := time.Parse(time.RFC3339Nano, report.Time)
if err != nil {
return
}
_, _ = s.Assurance.Append("snapshot-log", job.logPath, captured)
_, _ = s.Assurance.Append("snapshot-json", job.jsonPath, captured)
}
}
func enrichmentPaths(value map[string]any) []string {
rows, _ := value["files"].([]any)
paths := make([]string, 0, len(rows))
for _, row := range rows {
if item, ok := row.(map[string]any); ok {
if path, ok := item["path"].(string); ok && path != "" {
paths = append(paths, path)
}
}
}
return paths
}
func mergeSlow(report *Report, updates enrich.SlowResult) {
if report.Enrichment == nil {
return
}
if rows, ok := report.Enrichment["processes"].([]any); ok {
for _, row := range rows {
item, ok := row.(map[string]any)
if !ok {
continue
}
pid, _ := item["pid"].(float64)
if update := updates.Processes[int(pid)]; update != nil {
item["exe_sha256"] = update["exe_sha256"]
}
}
}
if rows, ok := report.Enrichment["files"].([]any); ok {
for _, row := range rows {
item, ok := row.(map[string]any)
if !ok {
continue
}
path, _ := item["path"].(string)
if update := updates.Files[path]; update != nil {
for key, value := range update {
item[key] = value
}
}
}
}
}
func (s *Store) collectProcesses(alerts []model.Alert) []ProcessDetail {
pids := alertPIDs(alerts)
result := make([]ProcessDetail, 0, len(pids))

View file

@ -130,6 +130,53 @@ func TestCaptureEventLinksMergedSnapshotToOneIncident(t *testing.T) {
}
}
func TestSlowEnrichmentAddsBoundedHashAndFileMetadata(t *testing.T) {
store, err := New(t.TempDir(), "/unused", 10, 30)
if err != nil {
t.Fatal(err)
}
store.EnableSlowEnrichment()
defer store.Close()
exe := filepath.Join(store.Dir, "payload")
watch := filepath.Join(store.Dir, "persistence.service")
if err := os.WriteFile(exe, []byte("stage-one"), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(watch, []byte("[Service]"), 0o600); err != nil {
t.Fatal(err)
}
store.Collect = func(pid int) ProcessDetail {
return ProcessDetail{PID: pid, Comm: "payload", Exe: exe, FDs: map[string]string{}}
}
event := alertEvent("2026-07-22T10:11:12Z", model.Alert{SID: 1, Severity: "HIGH", Signature: "test", Classtype: "test", Key: "file", Detail: "modified " + watch, PIDs: []int{42}})
if _, err := store.CaptureEvent(event); err != nil {
t.Fatal(err)
}
store.FlushSlow()
raw, err := os.ReadFile(filepath.Join(store.Dir, "alert-20260722-101112.json"))
if err != nil {
t.Fatal(err)
}
var report Report
if err := json.Unmarshal(raw, &report); err != nil {
t.Fatal(err)
}
processes := report.Enrichment["processes"].([]any)
process := processes[0].(map[string]any)
if digest, ok := process["exe_sha256"].(string); !ok || len(digest) != 64 {
t.Fatalf("process enrichment=%#v", process)
}
files := report.Enrichment["files"].([]any)
file := files[0].(map[string]any)
if file["exists"] != true || file["mode"] != float64(0o600) {
t.Fatalf("file enrichment=%#v", file)
}
logRaw, err := os.ReadFile(filepath.Join(store.Dir, "alert-20260722-101112.log"))
if err != nil || !strings.Contains(string(logRaw), "ENODIA SENTINEL ALERT") {
t.Fatalf("slow log err=%v text=%q", err, logRaw)
}
}
func TestRetentionPrunesCountAndAgeAsPairs(t *testing.T) {
store, err := New(t.TempDir(), "/unused", 2, 1)
if err != nil {