From 670d19e21c464af055c50bbc1208b887b8695190 Mon Sep 17 00:00:00 2001 From: Tri Lam Date: Mon, 1 Jun 2026 02:01:55 -0700 Subject: [PATCH 1/9] test(schema): RED v1.0-rc1 envelope test suite Adds verdict_envelope_schema_test.go pinning cut-criterion-2 contract before the artifact lands: schema loadable from docs/schemas/verdict- 1.0.0-rc1.json, every shipped Verdict round-trips through it, ten envelope-drift falsifiers reject, and $id matches the published CDN URL contract for external SDK pinning. Fails with file-not-found until the schema artifact is published. Signed-off-by: Tri Lam --- .../patterns/verdict_envelope_schema_test.go | 288 ++++++++++++++++++ 1 file changed, 288 insertions(+) create mode 100644 module/pkg/patterns/verdict_envelope_schema_test.go diff --git a/module/pkg/patterns/verdict_envelope_schema_test.go b/module/pkg/patterns/verdict_envelope_schema_test.go new file mode 100644 index 00000000..848b10e3 --- /dev/null +++ b/module/pkg/patterns/verdict_envelope_schema_test.go @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: Apache-2.0 + +package patterns_test + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/santhosh-tekuri/jsonschema/v6" + "github.com/stretchr/testify/require" + + "github.com/tracecoreai/tracecore/module/pkg/patterns" +) + +// envelopeSchemaPath is the repo-mirrored v1.0-rc1 Verdict envelope +// schema (cut criterion #2). It is intentionally one level up from +// module/ so it ships at docs/schemas/verdict-1.0.0-rc1.json — the +// path third-party consumers can pull from a tagged release without +// reaching into the Go module tree. +var envelopeSchemaPath = filepath.Join("..", "..", "..", "docs", "schemas", "verdict-1.0.0-rc1.json") + +// loadEnvelopeSchema compiles the v1.0-rc1 envelope schema and fails +// the test if the artifact is missing — a missing artifact is the +// cut-criterion-2 failure mode the schema-publication step is meant +// to defend against. +func loadEnvelopeSchema(t *testing.T) *jsonschema.Schema { + t.Helper() + bs, err := os.ReadFile(envelopeSchemaPath) //nolint:gosec // test-local relative path + require.NoError(t, err, + "docs/schemas/verdict-1.0.0-rc1.json must exist; this artifact is "+ + "cut-criterion-2 for v1.0-rc1 and is read by external verdict-"+ + "consumption SDKs (criterion 12). Run `git status docs/schemas/` "+ + "and verify the file landed.") + + compiler := jsonschema.NewCompiler() + var doc any + require.NoError(t, json.Unmarshal(bs, &doc)) + require.NoError(t, compiler.AddResource(envelopeSchemaPath, doc)) + schema, err := compiler.Compile(envelopeSchemaPath) + require.NoError(t, err) + return schema +} + +// validatesEnvelope marshals v, decodes back to a generic any, and +// asserts the schema accepts it. The marshal-then-decode round-trip +// is what an external consumer sees on the wire; validating the Go +// struct directly would skip JSON-tag drift. +func validatesEnvelope(t *testing.T, schema *jsonschema.Schema, v any) { + t.Helper() + bs, err := json.Marshal(v) + require.NoError(t, err) + var decoded any + require.NoError(t, json.Unmarshal(bs, &decoded)) + require.NoError(t, schema.Validate(decoded), + "%T failed v1.0-rc1 envelope validation; either the verdict "+ + "struct drifted or docs/schemas/verdict-1.0.0-rc1.json needs a "+ + "superset update (run through docs/DEPRECATION.md policy).", v) +} + +// TestVerdictEnvelopeV1RC1_AllShippedVerdictsValidate is the +// cut-criterion-2 cross-cutting test: every Verdict the patterns +// package ships today must round-trip through the published envelope +// schema. A new pattern added later MUST add a fixture here — the +// rubric is that the envelope is the union-superset, not a snapshot +// of the patterns shipped on the day rc1 was cut. +func TestVerdictEnvelopeV1RC1_AllShippedVerdictsValidate(t *testing.T) { + t.Parallel() + schema := loadEnvelopeSchema(t) + + ts := time.Date(2026, 5, 18, 10, 0, 0, 0, time.UTC) + ev := func(kind string) patterns.EvidenceRef { + return patterns.EvidenceRef{Kind: kind, UID: "u", Timestamp: ts, Description: "d"} + } + + // One fixture per shipped verdict. The fixture is the simplest + // well-formed Verdict the detector can emit; envelope-superset + // behavior is the only thing under test (per-pattern shape is + // covered by the per-pattern testdata schema files). + cases := []struct { + name string + verdict any + }{ + {"pod_evicted", patterns.PodEvictedVerdict{ + PatternID: patterns.PatternIDPodEvicted, + Headline: "h", Remediation: "r", + Confidence: patterns.ConfidenceFull, + EvidenceTrail: []patterns.EvidenceRef{ev(patterns.EvidenceKindPodEvent), ev(patterns.EvidenceKindNodeCondition)}, + }}, + {"nccl_hang", patterns.NCCLHangVerdict{ + PatternID: patterns.PatternIDNCCLHang, + Headline: "h", Remediation: "r", + EvidenceTrail: []patterns.EvidenceRef{ev(patterns.EvidenceKindNCCLFR)}, + PgID: 7, + CollectiveSeqID: 42, + HangingRanks: []int64{1, 3}, + }}, + {"xid_correlation", patterns.XidCorrelationVerdict{ + PatternID: patterns.PatternIDXidCorrelation, + Headline: "h", Remediation: "r", + EvidenceTrail: []patterns.EvidenceRef{ev(patterns.EvidenceKindXid), ev(patterns.EvidenceKindPodEvent)}, + XidCode: 79, Node: "node-a", EvictedPod: "ns/p", + }}, + {"hbm_ecc", patterns.HBMECCVerdict{ + PatternID: patterns.PatternIDHBMECC, + Headline: "h", Remediation: "r", + EvidenceTrail: []patterns.EvidenceRef{ev(patterns.EvidenceKindHBM), ev(patterns.EvidenceKindXid)}, + XidCode: 48, GPUID: "0000:31:00.0", ECCDelta: 1, Node: "node-a", + }}, + {"thermal_throttle", patterns.ThermalThrottleVerdict{ + PatternID: patterns.PatternIDThermalThrottle, + Headline: "h", Remediation: "r", + EvidenceTrail: []patterns.EvidenceRef{ev(patterns.EvidenceKindHWThrottle), ev(patterns.EvidenceKindHWThrottle)}, + Node: "node-a", GPUCount: 2, GPUIDs: []string{"0000:31:00.0", "0000:32:00.0"}, + }}, + {"pcie_aer", patterns.PCIeAERVerdict{ + PatternID: patterns.PatternIDPCIeAER, + Headline: "h", Remediation: "r", + EvidenceTrail: []patterns.EvidenceRef{ev(patterns.EvidenceKindAER), ev(patterns.EvidenceKindPCIeIOCollapse)}, + GPUID: "0000:31:00.0", Severity: "Fatal", AERType: "Data Link Layer", + DropRatio: 0.9, Node: "node-a", + }}, + {"cuda_oom", patterns.CUDAOOMVerdict{ + PatternID: patterns.PatternIDCUDAOOM, + Headline: "h", Remediation: "r", + Confidence: patterns.ConfidenceFull, + EvidenceTrail: []patterns.EvidenceRef{ev(patterns.EvidenceKindCUDAOOM), ev(patterns.EvidenceKindHwFB)}, + GPUID: "0000:31:00.0", Node: "node-a", + Kind: "true_oom", TriedAllocBytes: 1024, FBFreeBytes: 0, FBFreeRatio: 0, + }}, + {"ib_link_flap", patterns.IBLinkFlapVerdict{ + PatternID: patterns.PatternIDIBLinkFlap, + Headline: "h", Remediation: "r", + Confidence: patterns.ConfidencePartial, + EvidenceTrail: []patterns.EvidenceRef{ev(patterns.EvidenceKindIBPortState)}, + Node: "node-a", HCADevice: "mlx5_0", Port: 1, TransitionCount: 3, + }}, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + validatesEnvelope(t, schema, tc.verdict) + }) + } +} + +// TestVerdictEnvelopeV1RC1_RejectsEnvelopeDrift asserts the +// envelope's bite: removing/mangling any envelope-level required +// field fails validation. Each row is a one-mutation falsifier for +// one envelope guard; if a future schema loosening regresses, the +// matching row will flip to PASS and surface the regression. This +// mirrors the per-pattern drift test +// (TestPodEvictedVerdict_SchemaRejectsDrift) at the envelope tier. +func TestVerdictEnvelopeV1RC1_RejectsEnvelopeDrift(t *testing.T) { + t.Parallel() + schema := loadEnvelopeSchema(t) + + validEvidence := []any{map[string]any{ + "kind": "pod_event", "uid": "u", + "timestamp": "2026-05-18T10:00:00Z", "description": "d", + }} + + cases := []struct { + name string + mutation map[string]any + guardName string + }{ + { + name: "missing_pattern_id", + mutation: map[string]any{ + "headline": "h", "remediation": "r", + "evidence_trail": validEvidence, + }, + guardName: "envelope required: pattern.id", + }, + { + name: "missing_headline", + mutation: map[string]any{ + "pattern.id": "14", "remediation": "r", + "evidence_trail": validEvidence, + }, + guardName: "envelope required: headline", + }, + { + name: "missing_remediation", + mutation: map[string]any{ + "pattern.id": "14", "headline": "h", + "evidence_trail": validEvidence, + }, + guardName: "envelope required: remediation", + }, + { + name: "missing_evidence_trail", + mutation: map[string]any{ + "pattern.id": "14", "headline": "h", "remediation": "r", + }, + guardName: "envelope required: evidence_trail", + }, + { + name: "empty_evidence_trail", + mutation: map[string]any{ + "pattern.id": "14", "headline": "h", "remediation": "r", + "evidence_trail": []any{}, + }, + guardName: "evidence_trail minItems:1", + }, + { + name: "pattern_id_numeric_not_string", + mutation: map[string]any{ + "pattern.id": 14, "headline": "h", "remediation": "r", + "evidence_trail": validEvidence, + }, + guardName: "pattern.id type:string", + }, + { + name: "confidence_outside_enum", + mutation: map[string]any{ + "pattern.id": "14", "headline": "h", "remediation": "r", + "confidence": "unknown", + "evidence_trail": validEvidence, + }, + guardName: "confidence enum (full|partial) when present", + }, + { + name: "evidence_kind_missing", + mutation: map[string]any{ + "pattern.id": "14", "headline": "h", "remediation": "r", + "evidence_trail": []any{map[string]any{ + "uid": "u", "timestamp": "2026-05-18T10:00:00Z", "description": "d", + }}, + }, + guardName: "evidence_trail item required: kind", + }, + { + name: "evidence_uid_empty", + mutation: map[string]any{ + "pattern.id": "14", "headline": "h", "remediation": "r", + "evidence_trail": []any{map[string]any{ + "kind": "pod_event", "uid": "", + "timestamp": "2026-05-18T10:00:00Z", "description": "d", + }}, + }, + guardName: "evidence_trail item uid minLength:1", + }, + { + name: "headline_empty_string", + mutation: map[string]any{ + "pattern.id": "14", "headline": "", "remediation": "r", + "evidence_trail": validEvidence, + }, + guardName: "headline minLength:1", + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + require.Error(t, schema.Validate(tc.mutation), + "envelope schema must reject %s; guard %q regressed", + tc.name, tc.guardName) + }) + } +} + +// TestVerdictEnvelopeV1RC1_HasStableID pins the envelope's externally +// observable identifier. Cut criterion 2 says rc1 publishes a stable +// shape; the $id is the contract third-party SDKs (criterion 12) key +// off to detect "this is the v1.0-rc1 envelope". A change to $id is +// the only signal an SDK has that the envelope is a new version, so +// drift here is a release-tagging bug, not a schema cleanup. +func TestVerdictEnvelopeV1RC1_HasStableID(t *testing.T) { + t.Parallel() + bs, err := os.ReadFile(envelopeSchemaPath) //nolint:gosec // test-local relative path + require.NoError(t, err) + var doc map[string]any + require.NoError(t, json.Unmarshal(bs, &doc)) + + require.Equal(t, "https://json-schema.org/draft/2020-12/schema", doc["$schema"], + "envelope must use draft 2020-12, matching the per-pattern testdata schemas") + require.Equal(t, "https://schema.tracecore.io/verdict/1.0.0-rc1.json", doc["$id"], + "$id is the externally observable version contract; bumping requires the "+ + "deprecation policy (docs/DEPRECATION.md) and an SDK release pair (criterion 12)") +} From 9f7a4a8a621f39c2e02107bb100dae07447ca33f Mon Sep 17 00:00:00 2001 From: Tri Lam Date: Mon, 1 Jun 2026 02:07:31 -0700 Subject: [PATCH 2/9] feat(schema): publish Verdict v1.0-rc1 envelope schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Publishes docs/schemas/verdict-1.0.0-rc1.json — the canonical Verdict envelope every pattern detector emits. Per-pattern detectors layer pattern-specific fields on top (gpu_id, xid_code, kind, …); per-pattern testdata schemas pin those extensions. Envelope sets additionalProperties true so per-pattern fields validate without re-listing here. Closes v1-rc1 cut criterion 2: - Schema at docs/schemas/verdict-1.0.0-rc1.json, draft 2020-12, $id https://schema.tracecore.io/verdict/1.0.0-rc1.json (matches the CDN contract third-party SDKs key off). - Envelope-conformance + ten drift-falsifier rows already landed RED in the prior commit; this commit makes them GREEN. - docs/schemas/README.md documents versioning + evolution policy and cross-links the forthcoming docs/DEPRECATION.md (criterion 4). - docs/ATTRIBUTES.md cross-links the envelope from the pattern.verdict_json column. - docs/v1-rc1-cut-criteria.md flips criterion 2 from planned to shipped with a Landed pointer. Signed-off-by: Tri Lam --- docs/ATTRIBUTES.md | 9 +++ docs/schemas/README.md | 108 ++++++++++++++++++++++++++++ docs/schemas/verdict-1.0.0-rc1.json | 83 +++++++++++++++++++++ docs/v1-rc1-cut-criteria.md | 11 ++- 4 files changed, 208 insertions(+), 3 deletions(-) create mode 100644 docs/schemas/README.md create mode 100644 docs/schemas/verdict-1.0.0-rc1.json diff --git a/docs/ATTRIBUTES.md b/docs/ATTRIBUTES.md index d038a187..b270279f 100644 --- a/docs/ATTRIBUTES.md +++ b/docs/ATTRIBUTES.md @@ -51,6 +51,15 @@ verdict log record (one record per detected pattern). The five scalar keys are required; the JSON blob carries the full evidence trail for downstream consumers that want it. +The shape inside `pattern.verdict_json` is pinned by the **Verdict +envelope schema v1.0-rc1** — see +[`docs/schemas/verdict-1.0.0-rc1.json`](schemas/verdict-1.0.0-rc1.json) +(versioning + evolution policy: [`docs/schemas/README.md`](schemas/README.md)). +A rename of any envelope-level field goes through the deprecation +policy (forthcoming `docs/DEPRECATION.md`, tracked as +[cut criterion 4](v1-rc1-cut-criteria.md#4-deprecation-policy--enforcement-gate)) +rather than free drift. + | Attribute | Type | Source | Stability | Description | Emitted by | Consumed by | |---|---|---|---|---|---|---| | `pattern.id` | string | tracecore-ext | stable | Canonical pattern identifier (`pod_evicted`, `xid_correlation`, `hbm_ecc`, `nccl_hang`, `thermal_throttle`, `pcie_aer`, `ib_link_flap`, `cuda_oom`, `silent_data_corruption`) | `patterndetectorprocessor` (`VerdictAttrPatternID`) | Dashboards, LogQL filters, runbooks | diff --git a/docs/schemas/README.md b/docs/schemas/README.md new file mode 100644 index 00000000..de862576 --- /dev/null +++ b/docs/schemas/README.md @@ -0,0 +1,108 @@ +# `docs/schemas/` — externally observable schema artifacts + +This directory holds the JSON Schema / vocabulary artifacts third-party +consumers bind against. Each artifact has a stable filename keyed on a +released version; consumers pin to that filename rather than chasing +`main`. + +## Inventory + +| Artifact | Surface | Status | Pinned by | +|---|---|---|---| +| [`verdict-1.0.0-rc1.json`](verdict-1.0.0-rc1.json) | Pattern-detector verdict envelope (every pattern emits this shape; per-pattern fields layer on top) | `v1.0-rc1` — published for [v1-rc1 cut criterion 2](../v1-rc1-cut-criteria.md#2-verdict-schema-v10-published-and-stable) | [`module/pkg/patterns/verdict_envelope_schema_test.go`](../../module/pkg/patterns/verdict_envelope_schema_test.go) | +| [`nccl_fr/v0.md`](nccl_fr/v0.md) | NCCL FlightRecorder receiver — OTel attribute vocabulary on the emitted log records | `v0` — additive within major | `schema_url` on every ResourceLogs / ScopeLogs envelope the receiver emits | + +## Verdict envelope (`verdict-1.0.0-rc1.json`) + +The envelope is the union-superset of fields every pattern detector +emits today. Per-pattern detectors layer pattern-specific fields on top +(`gpu_id`, `xid_code`, `kind`, etc.); the per-pattern testdata schemas +under [`module/pkg/patterns/testdata/`](../../module/pkg/patterns/testdata/) +pin those extensions. The envelope sets `additionalProperties: true` +deliberately — drift in pattern-specific fields surfaces on the +per-pattern schema, drift in envelope-level fields surfaces here. + +The `$id` (`https://schema.tracecore.io/verdict/1.0.0-rc1.json`) is the +externally observable version contract. SDKs (see +[criterion 12](../v1-rc1-cut-criteria.md#12-verdict-consumption-sdks-python--go)) +key off `$id` to detect "this is the v1.0-rc1 envelope" — changing +`$id` is a major-version event, not a schema cleanup. + +### What's in the envelope vs the per-pattern shape + +| Field | Where pinned | +|---|---| +| `pattern.id`, `headline`, `remediation`, `evidence_trail`, `confidence`, `missing_layers` | Envelope (`verdict-1.0.0-rc1.json`) | +| `gpu_id`, `xid_code`, `severity`, `aer_type`, `kind`, `tried_alloc_bytes`, `fb_free_bytes`, `pg_id`, `collective_seq_id`, `hanging_ranks`, `gpu_count`, `gpu_ids`, `node`, `hca_device`, `port`, `transition_count`, `ecc_delta`, `evicted_pod`, `k8s.*` | Per-pattern testdata schemas | +| `evidence_trail[].kind` enum value | Envelope lists the union; per-pattern schemas tighten to the pattern's allowed subset | + +## Versioning policy + +The envelope filename carries the full version including the pre-release +qualifier (`verdict-1.0.0-rc1.json`, `verdict-1.0.0.json`, +`verdict-1.1.0.json`). Old versions stay on disk for the deprecation +window (forthcoming `docs/DEPRECATION.md`, tracked as +[cut criterion 4](../v1-rc1-cut-criteria.md#4-deprecation-policy--enforcement-gate)) +— never delete a published version. SDKs that pin to an old `$id` keep +working until the window elapses. + +### When to bump + +Use the same SemVer semantics as the Go module: + +- **Patch** (`1.0.0` → `1.0.1`) — clarify a description, fix a typo, no + shape change. Same `$id`. Consumers see the new file and the old file + with identical observable shape. +- **Minor** (`1.0.0` → `1.1.0`) — add an optional field, add an enum + value, loosen a constraint. New filename + new `$id`. Old consumers + continue to validate (their inputs are a subset of the new schema). +- **Major** (`1.0.0` → `2.0.0`) — remove a field, tighten a constraint, + change a type, change `additionalProperties` semantics. New filename + + new `$id`. Requires: + - A deprecation entry in the forthcoming `docs/DEPRECATION.md` + (tracked as + [cut criterion 4](../v1-rc1-cut-criteria.md#4-deprecation-policy--enforcement-gate)) + naming the removal target release and the migration path. + - A `### Deprecated` entry in the CHANGELOG for the release that + publishes the new shape. + - An updated SDK release pair ([criterion 12](../v1-rc1-cut-criteria.md#12-verdict-consumption-sdks-python--go)) + pinning to the new `$id`. + +The rc → final transition (`1.0.0-rc1` → `1.0.0`) follows the same +rule: if shape did not change between rc and final, the only delta is +the filename. If shape changed, the rc was incorrect and a new rc gets +cut. + +### How a new pattern adds itself + +1. Add the detector + its `Verdict` Go type under `module/pkg/patterns/`. +2. Add the per-pattern schema under + `module/pkg/patterns/testdata/_verdict.schema.json` + (envelope-conformant — every required field comes from the envelope). +3. Add the new EvidenceKind constant (if the detector introduces one). + The envelope's `evidence_trail[].kind` enum must list the value — + that is a minor-version bump (additive). +4. Add a fixture row to + `TestVerdictEnvelopeV1RC1_AllShippedVerdictsValidate` in + [`verdict_envelope_schema_test.go`](../../module/pkg/patterns/verdict_envelope_schema_test.go). +5. Cross-link from + [`docs/ATTRIBUTES.md`](../ATTRIBUTES.md#pattern--verdict-log-record-attributes). + +## How drift is caught + +| Drift class | Caught by | +|---|---| +| Envelope-level field added / removed / loosened in the Go type without a schema bump | `TestVerdictEnvelopeV1RC1_AllShippedVerdictsValidate` (every shipped Verdict round-trips through the published artifact) | +| Envelope guard removed in the schema (e.g. `headline` becomes optional, `confidence` enum loosens) | `TestVerdictEnvelopeV1RC1_RejectsEnvelopeDrift` (ten falsifier rows; loosening a guard flips its row to PASS) | +| `$id` changed without a major-version bump | `TestVerdictEnvelopeV1RC1_HasStableID` (pins the published `$id` literal) | +| Per-pattern field drift | per-pattern `TestVerdict_SchemaConformance` / `_SchemaRejectsDrift` | +| New EvidenceKind constant added without listing in envelope | The new pattern's fixture in `TestVerdictEnvelopeV1RC1_AllShippedVerdictsValidate` fails until the envelope enum is updated | + +## Hosting + +The repo-mirrored copy at `docs/schemas/verdict-1.0.0-rc1.json` is the +source of truth. The CDN URL the `$id` references +(`https://schema.tracecore.io/verdict/1.0.0-rc1.json`) is a mirror of +the same file at the rc1 tag — populated by the release pipeline at +tag-cut time. SDKs may fetch from either location; the byte-for-byte +content is identical. diff --git a/docs/schemas/verdict-1.0.0-rc1.json b/docs/schemas/verdict-1.0.0-rc1.json new file mode 100644 index 00000000..1fbaa251 --- /dev/null +++ b/docs/schemas/verdict-1.0.0-rc1.json @@ -0,0 +1,83 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schema.tracecore.io/verdict/1.0.0-rc1.json", + "title": "TraceCore Verdict envelope v1.0-rc1", + "description": "Canonical envelope shape every TraceCore pattern detector emits. This is the v1.0-rc1 cut of the externally observable verdict contract — the shape downstream consumers (Prometheus alertmanager webhooks, Slack routers, audit pipelines) bind against. Per-pattern detectors layer pattern-specific fields on top of this envelope (e.g. `gpu_id`, `xid_code`, `kind`); per-pattern testdata schemas under module/pkg/patterns/testdata/_verdict.schema.json pin those extensions. The envelope intentionally sets `additionalProperties: true` so per-pattern extensions validate without re-listing here — drift in pattern-specific fields surfaces on the per-pattern schema, drift in envelope-level fields surfaces here. Versioning policy: docs/schemas/README.md; evolution policy: docs/DEPRECATION.md (forthcoming).", + "type": "object", + "required": ["pattern.id", "headline", "remediation", "evidence_trail"], + "additionalProperties": true, + "properties": { + "pattern.id": { + "type": "string", + "minLength": 1, + "description": "String-typed numeric identifier (e.g. \"14\" for pod_evicted, \"15\" for nccl_hang). Uniform across pattern detectors per MILESTONES.md §M19 rubric L405. String-typed (not integer) so future namespacing (e.g. \"vendor.42\") does not break the wire format." + }, + "headline": { + "type": "string", + "minLength": 1, + "description": "One-line operator-facing summary. Detector-controlled prose. Suitable as an alert title." + }, + "remediation": { + "type": "string", + "minLength": 1, + "description": "Detector-controlled remediation hint. Suitable as alert body text. Detectors key the prose on the underlying root cause (e.g. for pod_evicted: disk → relocate.*NVMe; memory → reduce per-rank memory; pid → cap fork rate)." + }, + "confidence": { + "type": "string", + "enum": ["full", "partial"], + "description": "How complete the evidence trail is. `full` = every required evidence layer joined; `partial` = at least one layer missing (recorded by name in `missing_layers`). Optional at the envelope level — patterns whose emission rule is \"both layers joined or no verdict\" (hbm_ecc, pcie_aer, thermal_throttle, xid_correlation, nccl_hang) omit this field rather than constant-stamp `full`." + }, + "evidence_trail": { + "type": "array", + "minItems": 1, + "description": "Ordered list of evidence references that drove the verdict. Reads top-to-bottom as the operator-facing timeline. Per-pattern schemas tighten `minItems` (e.g. hbm_ecc requires 2 entries — one hw_error, one kernel_event).", + "items": { + "type": "object", + "required": ["kind", "uid", "timestamp", "description"], + "additionalProperties": false, + "properties": { + "kind": { + "type": "string", + "enum": [ + "pod_event", + "node_condition", + "nccl_fr", + "kernel_event", + "hw_error", + "hw_throttle", + "hw_io_collapse", + "hw_fb", + "cuda_oom", + "pcie_aer", + "ib_port_state" + ], + "description": "Receiver-side surface the evidence came from. Per-pattern schemas tighten this to a pattern-specific subset (e.g. pod_evicted: [pod_event, node_condition]; pcie_aer: [pcie_aer, hw_io_collapse]). New kinds added here MUST land alongside a new EvidenceKind* constant in module/pkg/patterns/ and a per-pattern schema update." + }, + "uid": { + "type": "string", + "minLength": 1, + "description": "Upstream identifier. For Kubernetes Events: Event.UID; for node conditions: tracecore-synthetic `{node-uid}/{pressure-kind}/{transition-unix-nano}`; for kernel/nccl/hw events: receiver-specific stable UID." + }, + "timestamp": { + "type": "string", + "format": "date-time", + "description": "Evidence wall-clock time (RFC 3339 / ISO 8601). For k8s Events: Event.EventTime; for node conditions: LastTransitionTime." + }, + "description": { + "type": "string", + "minLength": 1, + "description": "One-line operator-facing summary. Detector-controlled prose; not parsed by anything downstream." + } + } + } + }, + "missing_layers": { + "type": "array", + "description": "Names of evidence layers that did not join, populated in causal order. Empty / omitted when `confidence` is `full` (or when the pattern does not carry `confidence`). Vocabulary is shared with `evidence_trail[].kind` — a missing layer is named by the kind it would have carried.", + "items": { + "type": "string", + "minLength": 1 + } + } + } +} diff --git a/docs/v1-rc1-cut-criteria.md b/docs/v1-rc1-cut-criteria.md index 07ba543d..30d4723c 100644 --- a/docs/v1-rc1-cut-criteria.md +++ b/docs/v1-rc1-cut-criteria.md @@ -62,9 +62,14 @@ Status is set against the repo state as of 2026-05-31. - **Citation:** [`docs/RELEASE-CHECKLIST.md`](RELEASE-CHECKLIST.md) RC additional gates ("Verdict schema v1.0 RC published") and [`NORTHSTARS.md` §O4](NORTHSTARS.md#o4-standards). -- **Status:** ☐ — `module/pkg/patterns/verdict.go` defines the in-Go - type today; the externally hostable JSON schema artifact has not - been published. +- **Status:** ☑ — repo-mirrored at + [`docs/schemas/verdict-1.0.0-rc1.json`](schemas/verdict-1.0.0-rc1.json) + with `$id` pointing at the CDN URL; envelope-conformance and + drift-falsifier tests at + [`module/pkg/patterns/verdict_envelope_schema_test.go`](../module/pkg/patterns/verdict_envelope_schema_test.go); + versioning + evolution policy at + [`docs/schemas/README.md`](schemas/README.md). **Landed:** PR + [`feat/verdict-schema-v1-rc1`](https://github.com/TraceCoreAI/tracecore/pulls?q=feat%2Fverdict-schema-v1-rc1). ### 3. Attribute namespace hard-locked From 35c07d1644fcd3207435cb289602546c4fd4c251 Mon Sep 17 00:00:00 2001 From: Tri Lam Date: Mon, 1 Jun 2026 02:19:26 -0700 Subject: [PATCH 3/9] =?UTF-8?q?test(sdk/verdict):=20RED=20=E2=80=94=20Go?= =?UTF-8?q?=20verdict-consumption=20SDK=20decoder?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drives criterion 12 (verdict-consumption SDKs) for v1-rc1. Tests fail with 'no non-test Go files' until verdict.go + decode.go land. Signed-off-by: Tri Lam --- module/sdk/verdict/decode_test.go | 265 ++++++++++++++++++++++++++++++ 1 file changed, 265 insertions(+) create mode 100644 module/sdk/verdict/decode_test.go diff --git a/module/sdk/verdict/decode_test.go b/module/sdk/verdict/decode_test.go new file mode 100644 index 00000000..c363c682 --- /dev/null +++ b/module/sdk/verdict/decode_test.go @@ -0,0 +1,265 @@ +// SPDX-License-Identifier: Apache-2.0 + +package verdict_test + +import ( + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/tracecoreai/tracecore/module/sdk/verdict" +) + +// goodVerdict is the canonical happy-path fixture — minimum-well-formed +// pod_evicted verdict shaped per the published envelope. +const goodVerdict = `{ + "pattern.id": "14", + "headline": "Pod ns/p evicted at 2026-05-18T10:00:00Z due to disk pressure", + "remediation": "relocate workload to a node with NVMe-backed ephemeral storage", + "confidence": "full", + "evidence_trail": [ + {"kind": "pod_event", "uid": "evt-1", "timestamp": "2026-05-18T10:00:00Z", "description": "Evicted: disk pressure"}, + {"kind": "node_condition", "uid": "node-a/disk/1716030000000000000", "timestamp": "2026-05-18T09:59:55Z", "description": "DiskPressure transitioned true"} + ] +}` + +// TestDecode_HappyPath: a known-good v1.0-rc1 envelope decodes into a +// typed Verdict with every field populated. +func TestDecode_HappyPath(t *testing.T) { + t.Parallel() + v, err := verdict.Decode([]byte(goodVerdict)) + require.NoError(t, err) + + require.Equal(t, "14", v.PatternID) + require.Contains(t, v.Headline, "Pod ns/p evicted") + require.Contains(t, v.Remediation, "NVMe") + require.Equal(t, verdict.ConfidenceFull, v.Confidence) + require.Len(t, v.EvidenceTrail, 2) + + require.Equal(t, "pod_event", v.EvidenceTrail[0].Kind) + require.Equal(t, "evt-1", v.EvidenceTrail[0].UID) + require.Equal(t, + time.Date(2026, 5, 18, 10, 0, 0, 0, time.UTC), + v.EvidenceTrail[0].Timestamp) + require.Equal(t, "Evicted: disk pressure", v.EvidenceTrail[0].Description) + + require.Equal(t, "node_condition", v.EvidenceTrail[1].Kind) +} + +// TestDecode_RejectsSchemaInvalid: the decoder is gated on schema +// validation. A required envelope-level field missing surfaces as an +// error, not a half-populated struct. This is the SDK's load-bearing +// promise — downstream consumers see typed verdicts only when the +// payload conforms to the published envelope. +func TestDecode_RejectsSchemaInvalid(t *testing.T) { + t.Parallel() + cases := []struct { + name string + raw string + }{ + { + name: "missing_pattern_id", + raw: `{"headline": "h", "remediation": "r", "evidence_trail": [ + {"kind": "pod_event", "uid": "u", "timestamp": "2026-05-18T10:00:00Z", "description": "d"} + ]}`, + }, + { + name: "missing_headline", + raw: `{"pattern.id": "14", "remediation": "r", "evidence_trail": [ + {"kind": "pod_event", "uid": "u", "timestamp": "2026-05-18T10:00:00Z", "description": "d"} + ]}`, + }, + { + name: "empty_evidence_trail", + raw: `{"pattern.id": "14", "headline": "h", "remediation": "r", "evidence_trail": []}`, + }, + { + name: "pattern_id_numeric_not_string", + raw: `{"pattern.id": 14, "headline": "h", "remediation": "r", "evidence_trail": [ + {"kind": "pod_event", "uid": "u", "timestamp": "2026-05-18T10:00:00Z", "description": "d"} + ]}`, + }, + { + name: "confidence_outside_enum", + raw: `{"pattern.id": "14", "headline": "h", "remediation": "r", "confidence": "maybe", "evidence_trail": [ + {"kind": "pod_event", "uid": "u", "timestamp": "2026-05-18T10:00:00Z", "description": "d"} + ]}`, + }, + { + name: "evidence_uid_empty", + raw: `{"pattern.id": "14", "headline": "h", "remediation": "r", "evidence_trail": [ + {"kind": "pod_event", "uid": "", "timestamp": "2026-05-18T10:00:00Z", "description": "d"} + ]}`, + }, + } + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + _, err := verdict.Decode([]byte(tc.raw)) + require.Error(t, err, + "Decode must reject schema-invalid input %q; this defends the "+ + "SDK's typed-output contract.", tc.name) + }) + } +} + +// TestDecode_RejectsMalformedJSON: a non-JSON body is rejected before +// schema validation runs. The error must be returned, not panicked. +func TestDecode_RejectsMalformedJSON(t *testing.T) { + t.Parallel() + _, err := verdict.Decode([]byte("{not json")) + require.Error(t, err) +} + +// TestDecode_PreservesExtras: the envelope sets +// `additionalProperties: true`, so per-pattern fields layered on top +// (k8s.pod.name, gpu_id, xid_code, kind, …) MUST survive the decode +// round-trip. Without this, the SDK would silently drop every +// per-pattern extension — making the typed Verdict useless for the +// pattern-specific routing rules downstream consumers care about. +func TestDecode_PreservesExtras(t *testing.T) { + t.Parallel() + raw := `{ + "pattern.id": "14", + "headline": "h", + "remediation": "r", + "confidence": "full", + "evidence_trail": [ + {"kind": "pod_event", "uid": "u", "timestamp": "2026-05-18T10:00:00Z", "description": "d"} + ], + "k8s.pod.name": "trainer-3", + "k8s.pod.namespace": "ml-prod", + "k8s.node.name": "node-a", + "some_future_field": {"nested": [1, 2, 3]} + }` + v, err := verdict.Decode([]byte(raw)) + require.NoError(t, err) + + require.Equal(t, "trainer-3", v.Extras["k8s.pod.name"]) + require.Equal(t, "ml-prod", v.Extras["k8s.pod.namespace"]) + require.Equal(t, "node-a", v.Extras["k8s.node.name"]) + + nested, ok := v.Extras["some_future_field"].(map[string]any) + require.True(t, ok, "nested extras decoded as map[string]any") + require.NotNil(t, nested["nested"]) + + // And the envelope-level fields are NOT duplicated into Extras — + // Extras carries only fields the typed struct does not bind. + _, headlineInExtras := v.Extras["headline"] + require.False(t, headlineInExtras, + "typed envelope fields must not leak into Extras") +} + +// TestDecode_AllShippedPatternsRoundTrip: every pattern verdict +// shipped at v1.0-rc1 must decode cleanly. This locks the SDK to the +// envelope-superset contract: adding a new pattern that breaks this +// case means the envelope itself has drifted — and the test is the +// canary. +func TestDecode_AllShippedPatternsRoundTrip(t *testing.T) { + t.Parallel() + + ts := "2026-05-18T10:00:00Z" + ev := func(kind string) map[string]any { + return map[string]any{ + "kind": kind, "uid": "u", + "timestamp": ts, "description": "d", + } + } + + cases := []struct { + name string + verdict map[string]any + }{ + {"pod_evicted", map[string]any{ + "pattern.id": "14", "headline": "h", "remediation": "r", + "confidence": "full", + "evidence_trail": []any{ev("pod_event"), ev("node_condition")}, + "k8s.pod.name": "p", "k8s.pod.namespace": "ns", "k8s.node.name": "n", + }}, + {"nccl_hang", map[string]any{ + "pattern.id": "15", "headline": "h", "remediation": "r", + "evidence_trail": []any{ev("nccl_fr")}, + "pg_id": float64(7), + "collective_seq_id": float64(42), + "hanging_ranks": []any{float64(1), float64(3)}, + }}, + {"xid_correlation", map[string]any{ + "pattern.id": "16", "headline": "h", "remediation": "r", + "evidence_trail": []any{ev("kernel_event"), ev("pod_event")}, + "xid_code": float64(79), "node": "n", "evicted_pod": "ns/p", + }}, + {"hbm_ecc", map[string]any{ + "pattern.id": "17", "headline": "h", "remediation": "r", + "evidence_trail": []any{ev("hw_error"), ev("kernel_event")}, + "xid_code": float64(48), "gpu_id": "0000:31:00.0", + "ecc_delta": float64(1), "node": "n", + }}, + {"thermal_throttle", map[string]any{ + "pattern.id": "18", "headline": "h", "remediation": "r", + "evidence_trail": []any{ev("hw_throttle"), ev("hw_throttle")}, + "node": "n", "gpu_count": float64(2), + "gpu_ids": []any{"0000:31:00.0", "0000:32:00.0"}, + }}, + {"pcie_aer", map[string]any{ + "pattern.id": "19", "headline": "h", "remediation": "r", + "evidence_trail": []any{ev("pcie_aer"), ev("hw_io_collapse")}, + "gpu_id": "0000:31:00.0", "severity": "Fatal", + "aer_type": "Data Link Layer", "drop_ratio": 0.9, "node": "n", + }}, + {"cuda_oom", map[string]any{ + "pattern.id": "20", "headline": "h", "remediation": "r", + "confidence": "full", + "evidence_trail": []any{ev("cuda_oom"), ev("hw_fb")}, + "gpu_id": "0000:31:00.0", "node": "n", + "kind": "true_oom", "tried_alloc_bytes": float64(1024), + "fb_free_bytes": float64(0), "fb_free_ratio": 0.0, + }}, + {"ib_link_flap", map[string]any{ + "pattern.id": "21", "headline": "h", "remediation": "r", + "confidence": "partial", + "evidence_trail": []any{ev("ib_port_state")}, + "missing_layers": []any{"hw_throttle"}, + "node": "n", "hca_device": "mlx5_0", + "port": float64(1), + "transition_count": float64(3), + }}, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + raw, err := json.Marshal(tc.verdict) + require.NoError(t, err) + + v, err := verdict.Decode(raw) + require.NoError(t, err, + "shipped pattern %q must decode cleanly through the envelope SDK", tc.name) + require.Equal(t, tc.verdict["pattern.id"], v.PatternID) + require.NotEmpty(t, v.EvidenceTrail) + }) + } +} + +// TestDecode_MissingLayersPopulated: when a verdict carries +// missing_layers, the typed struct surfaces them. +func TestDecode_MissingLayersPopulated(t *testing.T) { + t.Parallel() + raw := `{ + "pattern.id": "14", + "headline": "h", + "remediation": "r", + "confidence": "partial", + "evidence_trail": [ + {"kind": "pod_event", "uid": "u", "timestamp": "2026-05-18T10:00:00Z", "description": "d"} + ], + "missing_layers": ["node_condition"] + }` + v, err := verdict.Decode([]byte(raw)) + require.NoError(t, err) + require.Equal(t, verdict.ConfidencePartial, v.Confidence) + require.Equal(t, []string{"node_condition"}, v.MissingLayers) +} From 084deef9a91baaa30b9aab6272deb7cbba89df2a Mon Sep 17 00:00:00 2001 From: Tri Lam Date: Mon, 1 Jun 2026 02:22:50 -0700 Subject: [PATCH 4/9] =?UTF-8?q?feat(sdk/verdict):=20GREEN=20=E2=80=94=20Go?= =?UTF-8?q?=20verdict-consumption=20SDK?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Publishes module/sdk/verdict — the typed Go client for the v1.0-rc1 Verdict envelope. Decode validates against the embedded copy of docs/schemas/verdict-1.0.0-rc1.json; a sync test pins byte-equality with the canonical source. Per-pattern extensions ride through Verdict.Extras (envelope additionalProperties:true). Drives v1-rc1 cut criterion 12 (Go half). Signed-off-by: Tri Lam --- module/sdk/verdict/README.md | 105 +++++++++++++++++++++++ module/sdk/verdict/decode.go | 114 +++++++++++++++++++++++++ module/sdk/verdict/doc.go | 8 ++ module/sdk/verdict/example_test.go | 42 +++++++++ module/sdk/verdict/schema.json | 83 ++++++++++++++++++ module/sdk/verdict/schema_sync_test.go | 40 +++++++++ module/sdk/verdict/verdict.go | 97 +++++++++++++++++++++ 7 files changed, 489 insertions(+) create mode 100644 module/sdk/verdict/README.md create mode 100644 module/sdk/verdict/decode.go create mode 100644 module/sdk/verdict/doc.go create mode 100644 module/sdk/verdict/example_test.go create mode 100644 module/sdk/verdict/schema.json create mode 100644 module/sdk/verdict/schema_sync_test.go create mode 100644 module/sdk/verdict/verdict.go diff --git a/module/sdk/verdict/README.md b/module/sdk/verdict/README.md new file mode 100644 index 00000000..c0459c58 --- /dev/null +++ b/module/sdk/verdict/README.md @@ -0,0 +1,105 @@ +# TraceCore Verdict SDK (Go) + +Client-side SDK for consuming **TraceCore Verdict v1.0-rc1** envelopes. + +This is the Go half of v1.0-rc1 cut-criterion 12 ("Verdict-consumption +SDKs"). The Python sibling lives at [`python/tracecore_verdict/`](../../../python/tracecore_verdict). + +## Install + +```bash +go get github.com/tracecoreai/tracecore/module/sdk/verdict +``` + +The SDK lives inside the main `tracecore` module — no separate +`go.mod`. Tag the consuming version against the `v1.0.0-rc1` repo tag. + +## Use + +```go +package main + +import ( + "fmt" + "log" + + "github.com/tracecoreai/tracecore/module/sdk/verdict" +) + +func main() { + raw := []byte(`{ + "pattern.id": "14", + "headline": "Pod ml-prod/trainer-3 evicted at 2026-05-18T10:00:00Z due to disk pressure", + "remediation": "relocate workload to a node with NVMe-backed ephemeral storage", + "confidence": "full", + "evidence_trail": [ + {"kind": "pod_event", "uid": "evt-1", + "timestamp": "2026-05-18T10:00:00Z", + "description": "Evicted: disk pressure"} + ], + "k8s.pod.name": "trainer-3" + }`) + + v, err := verdict.Decode(raw) + if err != nil { + log.Fatal(err) + } + fmt.Println("pattern.id:", v.PatternID) + fmt.Println("headline:", v.Headline) + fmt.Println("confidence:", v.Confidence) + if pod, ok := v.Extras["k8s.pod.name"].(string); ok { + fmt.Println("pod:", pod) + } +} +``` + +## What's in the box + +| Symbol | Purpose | +| --- | --- | +| `Verdict` | Typed envelope: `PatternID`, `Headline`, `Remediation`, `Confidence`, `EvidenceTrail`, `MissingLayers`, `Extras` | +| `EvidenceRef` | One row in the evidence trail (`Kind`, `UID`, `Timestamp`, `Description`) | +| `Confidence` (`ConfidenceFull` \| `ConfidencePartial`) | Join-completeness signal | +| `Decode(rawJSON []byte) (*Verdict, error)` | JSON → schema-validated typed Verdict | + +## Pattern-specific fields + +The envelope sets `additionalProperties: true`, so every per-pattern +extension (`gpu_id`, `xid_code`, `kind`, `k8s.pod.name`, …) rides +through on `Verdict.Extras` (`map[string]any`). Use Go type assertions +to read them: + +```go +if pod, ok := v.Extras["k8s.pod.name"].(string); ok { /* … */ } +if xid, ok := v.Extras["xid_code"].(float64); ok { /* … */ } // JSON numbers +``` + +## Versioning + +This package targets **schema v1.0-rc1 only**. + +A future envelope revision lands as a new sibling package (e.g. +`module/sdk/verdict/v2`) — never a silent breaking change here. The +SDK's `schema.json` is byte-pinned against +`docs/schemas/verdict-1.0.0-rc1.json` by +`TestEmbeddedSchemaMatchesCanonical`; drift fails the build. + +## Errors + +`Decode` returns an error (not a panic) for: + +- malformed JSON +- missing envelope-required field (`pattern.id`, `headline`, + `remediation`, `evidence_trail`) +- `confidence` outside the `full|partial` enum +- empty `evidence_trail` +- evidence-row missing `kind` / `uid` / `timestamp` / `description` +- any other v1.0-rc1 envelope violation + +Per-pattern field constraints (e.g. `pcie_aer.severity ∈ {Fatal, +Non-Fatal, Corrected}`) are out of scope — those live on per-pattern +schemas under `module/pkg/patterns/testdata/`. + +## License + +Apache-2.0. diff --git a/module/sdk/verdict/decode.go b/module/sdk/verdict/decode.go new file mode 100644 index 00000000..e775a535 --- /dev/null +++ b/module/sdk/verdict/decode.go @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: Apache-2.0 + +package verdict + +import ( + _ "embed" + "encoding/json" + "errors" + "fmt" + "sync" + + "github.com/santhosh-tekuri/jsonschema/v6" +) + +// envelopeSchemaJSON is the canonical v1.0-rc1 envelope schema, +// embedded at build time so the SDK ships as a single binary with no +// runtime filesystem dependency. The source of truth lives at +// docs/schemas/verdict-1.0.0-rc1.json; the embed path resolves through +// the symlink schema.json checked in alongside this file. +// +//go:embed schema.json +var envelopeSchemaJSON []byte + +// typedFields is the set of envelope-level properties that bind to +// dedicated [Verdict] struct fields. Decode strips these from the +// raw object before populating [Verdict.Extras] — extras carry only +// per-pattern extensions, not duplicate copies of the typed envelope. +var typedFields = map[string]struct{}{ + "pattern.id": {}, + "headline": {}, + "remediation": {}, + "confidence": {}, + "evidence_trail": {}, + "missing_layers": {}, +} + +// compiledSchema memoizes the compiled envelope schema. Compilation is +// done lazily on first Decode so importing the package has zero cost. +var ( + compiledSchemaOnce sync.Once + compiledSchema *jsonschema.Schema + compiledSchemaErr error +) + +// loadSchema compiles the embedded envelope schema. The compiled +// artifact is reused across every Decode call. +func loadSchema() (*jsonschema.Schema, error) { + compiledSchemaOnce.Do(func() { + var doc any + if err := json.Unmarshal(envelopeSchemaJSON, &doc); err != nil { + compiledSchemaErr = fmt.Errorf("verdict-sdk: parse embedded schema: %w", err) + return + } + c := jsonschema.NewCompiler() + const schemaURL = "https://schema.tracecore.io/verdict/1.0.0-rc1.json" + if err := c.AddResource(schemaURL, doc); err != nil { + compiledSchemaErr = fmt.Errorf("verdict-sdk: register embedded schema: %w", err) + return + } + s, err := c.Compile(schemaURL) + if err != nil { + compiledSchemaErr = fmt.Errorf("verdict-sdk: compile embedded schema: %w", err) + return + } + compiledSchema = s + }) + return compiledSchema, compiledSchemaErr +} + +// Decode validates rawJSON against the v1.0-rc1 envelope schema and +// returns a typed [Verdict]. Returns an error if the payload is not +// JSON or does not conform to the envelope (missing required fields, +// wrong type, confidence outside enum, …). +// +// Per-pattern fields (anything the envelope does not bind to a typed +// struct field) survive on [Verdict.Extras]. The envelope's +// `additionalProperties: true` guarantees this. +func Decode(rawJSON []byte) (*Verdict, error) { + schema, err := loadSchema() + if err != nil { + return nil, err + } + + var generic any + if err := json.Unmarshal(rawJSON, &generic); err != nil { + return nil, fmt.Errorf("verdict-sdk: malformed JSON: %w", err) + } + if err := schema.Validate(generic); err != nil { + return nil, fmt.Errorf("verdict-sdk: schema validation failed: %w", err) + } + + var v Verdict + if err := json.Unmarshal(rawJSON, &v); err != nil { + return nil, fmt.Errorf("verdict-sdk: decode typed envelope: %w", err) + } + + // Capture every non-typed field into Extras. The schema validate + // above guarantees the payload is a top-level object, so this + // type assertion is safe. + raw, ok := generic.(map[string]any) + if !ok { + return nil, errors.New("verdict-sdk: envelope root is not an object") + } + extras := make(map[string]any, len(raw)) + for k, val := range raw { + if _, typed := typedFields[k]; typed { + continue + } + extras[k] = val + } + v.Extras = extras + + return &v, nil +} diff --git a/module/sdk/verdict/doc.go b/module/sdk/verdict/doc.go new file mode 100644 index 00000000..4c35a1fb --- /dev/null +++ b/module/sdk/verdict/doc.go @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Doc-only file. The package doc-comment lives on verdict.go so +// `godoc` surfaces a single canonical description; this file is the +// home for the runnable example below (visible on pkg.go.dev under +// the "Examples" tab). + +package verdict diff --git a/module/sdk/verdict/example_test.go b/module/sdk/verdict/example_test.go new file mode 100644 index 00000000..77764e2e --- /dev/null +++ b/module/sdk/verdict/example_test.go @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: Apache-2.0 + +package verdict_test + +import ( + "fmt" + + "github.com/tracecoreai/tracecore/module/sdk/verdict" +) + +// ExampleDecode is the canonical operator-facing entry point: decode +// the `pattern.verdict_json` attribute payload into a typed Verdict +// and read per-pattern extensions through Extras. +func ExampleDecode() { + raw := []byte(`{ + "pattern.id": "14", + "headline": "Pod ml-prod/trainer-3 evicted at 2026-05-18T10:00:00Z due to disk pressure", + "remediation": "relocate workload to a node with NVMe-backed ephemeral storage", + "confidence": "full", + "evidence_trail": [ + {"kind": "pod_event", "uid": "evt-1", + "timestamp": "2026-05-18T10:00:00Z", + "description": "Evicted: disk pressure"} + ], + "k8s.pod.name": "trainer-3" + }`) + + v, err := verdict.Decode(raw) + if err != nil { + fmt.Println("decode failed:", err) + return + } + fmt.Println("pattern.id:", v.PatternID) + fmt.Println("confidence:", v.Confidence) + if pod, ok := v.Extras["k8s.pod.name"].(string); ok { + fmt.Println("pod:", pod) + } + // Output: + // pattern.id: 14 + // confidence: full + // pod: trainer-3 +} diff --git a/module/sdk/verdict/schema.json b/module/sdk/verdict/schema.json new file mode 100644 index 00000000..1fbaa251 --- /dev/null +++ b/module/sdk/verdict/schema.json @@ -0,0 +1,83 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schema.tracecore.io/verdict/1.0.0-rc1.json", + "title": "TraceCore Verdict envelope v1.0-rc1", + "description": "Canonical envelope shape every TraceCore pattern detector emits. This is the v1.0-rc1 cut of the externally observable verdict contract — the shape downstream consumers (Prometheus alertmanager webhooks, Slack routers, audit pipelines) bind against. Per-pattern detectors layer pattern-specific fields on top of this envelope (e.g. `gpu_id`, `xid_code`, `kind`); per-pattern testdata schemas under module/pkg/patterns/testdata/_verdict.schema.json pin those extensions. The envelope intentionally sets `additionalProperties: true` so per-pattern extensions validate without re-listing here — drift in pattern-specific fields surfaces on the per-pattern schema, drift in envelope-level fields surfaces here. Versioning policy: docs/schemas/README.md; evolution policy: docs/DEPRECATION.md (forthcoming).", + "type": "object", + "required": ["pattern.id", "headline", "remediation", "evidence_trail"], + "additionalProperties": true, + "properties": { + "pattern.id": { + "type": "string", + "minLength": 1, + "description": "String-typed numeric identifier (e.g. \"14\" for pod_evicted, \"15\" for nccl_hang). Uniform across pattern detectors per MILESTONES.md §M19 rubric L405. String-typed (not integer) so future namespacing (e.g. \"vendor.42\") does not break the wire format." + }, + "headline": { + "type": "string", + "minLength": 1, + "description": "One-line operator-facing summary. Detector-controlled prose. Suitable as an alert title." + }, + "remediation": { + "type": "string", + "minLength": 1, + "description": "Detector-controlled remediation hint. Suitable as alert body text. Detectors key the prose on the underlying root cause (e.g. for pod_evicted: disk → relocate.*NVMe; memory → reduce per-rank memory; pid → cap fork rate)." + }, + "confidence": { + "type": "string", + "enum": ["full", "partial"], + "description": "How complete the evidence trail is. `full` = every required evidence layer joined; `partial` = at least one layer missing (recorded by name in `missing_layers`). Optional at the envelope level — patterns whose emission rule is \"both layers joined or no verdict\" (hbm_ecc, pcie_aer, thermal_throttle, xid_correlation, nccl_hang) omit this field rather than constant-stamp `full`." + }, + "evidence_trail": { + "type": "array", + "minItems": 1, + "description": "Ordered list of evidence references that drove the verdict. Reads top-to-bottom as the operator-facing timeline. Per-pattern schemas tighten `minItems` (e.g. hbm_ecc requires 2 entries — one hw_error, one kernel_event).", + "items": { + "type": "object", + "required": ["kind", "uid", "timestamp", "description"], + "additionalProperties": false, + "properties": { + "kind": { + "type": "string", + "enum": [ + "pod_event", + "node_condition", + "nccl_fr", + "kernel_event", + "hw_error", + "hw_throttle", + "hw_io_collapse", + "hw_fb", + "cuda_oom", + "pcie_aer", + "ib_port_state" + ], + "description": "Receiver-side surface the evidence came from. Per-pattern schemas tighten this to a pattern-specific subset (e.g. pod_evicted: [pod_event, node_condition]; pcie_aer: [pcie_aer, hw_io_collapse]). New kinds added here MUST land alongside a new EvidenceKind* constant in module/pkg/patterns/ and a per-pattern schema update." + }, + "uid": { + "type": "string", + "minLength": 1, + "description": "Upstream identifier. For Kubernetes Events: Event.UID; for node conditions: tracecore-synthetic `{node-uid}/{pressure-kind}/{transition-unix-nano}`; for kernel/nccl/hw events: receiver-specific stable UID." + }, + "timestamp": { + "type": "string", + "format": "date-time", + "description": "Evidence wall-clock time (RFC 3339 / ISO 8601). For k8s Events: Event.EventTime; for node conditions: LastTransitionTime." + }, + "description": { + "type": "string", + "minLength": 1, + "description": "One-line operator-facing summary. Detector-controlled prose; not parsed by anything downstream." + } + } + } + }, + "missing_layers": { + "type": "array", + "description": "Names of evidence layers that did not join, populated in causal order. Empty / omitted when `confidence` is `full` (or when the pattern does not carry `confidence`). Vocabulary is shared with `evidence_trail[].kind` — a missing layer is named by the kind it would have carried.", + "items": { + "type": "string", + "minLength": 1 + } + } + } +} diff --git a/module/sdk/verdict/schema_sync_test.go b/module/sdk/verdict/schema_sync_test.go new file mode 100644 index 00000000..a147438c --- /dev/null +++ b/module/sdk/verdict/schema_sync_test.go @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: Apache-2.0 + +package verdict_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestEmbeddedSchemaMatchesCanonical pins the embedded schema.json +// against the canonical docs/schemas/verdict-1.0.0-rc1.json that +// criterion 2 publishes. The Go `//go:embed` directive cannot +// traverse parent directories, so the SDK ships a byte-identical +// copy. If the canonical schema drifts, this test fails and forces +// the operator to either: +// +// cp docs/schemas/verdict-1.0.0-rc1.json module/sdk/verdict/schema.json +// +// or, if the change is intentional, bump the SDK to a new schema +// version (a new sibling package — never a silent SDK shape change). +// +// This is the single-source-of-truth guard the criterion-12 brief +// requires. +func TestEmbeddedSchemaMatchesCanonical(t *testing.T) { + t.Parallel() + + embedded, err := os.ReadFile(filepath.Join(".", "schema.json")) //nolint:gosec // test-local + require.NoError(t, err, "module/sdk/verdict/schema.json must exist (build-time embed source)") + + canonical, err := os.ReadFile( + filepath.Join("..", "..", "..", "docs", "schemas", "verdict-1.0.0-rc1.json")) //nolint:gosec // test-local + require.NoError(t, err, "docs/schemas/verdict-1.0.0-rc1.json is the criterion-2 source of truth") + + require.Equal(t, string(canonical), string(embedded), + "module/sdk/verdict/schema.json drifted from docs/schemas/verdict-1.0.0-rc1.json. "+ + "Run: cp docs/schemas/verdict-1.0.0-rc1.json module/sdk/verdict/schema.json") +} diff --git a/module/sdk/verdict/verdict.go b/module/sdk/verdict/verdict.go new file mode 100644 index 00000000..8c2063f4 --- /dev/null +++ b/module/sdk/verdict/verdict.go @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package verdict is the client-side SDK for consuming TraceCore +// Verdict envelopes (v1.0-rc1). It exposes a typed [Verdict] plus a +// [Decode] entrypoint that gates output on the published envelope +// schema (docs/schemas/verdict-1.0.0-rc1.json). +// +// This package is the binding contract for criterion 12 of the v1.0-rc1 +// cut. It is intentionally a thin client-facing surface — a subset of +// the producer-side types in module/pkg/patterns. Producer-side fields +// the operator cannot use to route alerts (internal tags, debug hooks) +// are excluded; per-pattern extensions ride through [Verdict.Extras] +// because the envelope sets `additionalProperties: true`. +// +// Versioning: this package targets schema v1.0-rc1 only. A future +// envelope revision lands as a new sibling package (e.g. +// module/sdk/verdict/v2) — never a breaking change here. +package verdict + +import "time" + +// Confidence enumerates how complete a verdict's evidence trail is. +// `Full` means every required evidence layer joined; `Partial` means +// at least one was missing (recorded by name in [Verdict.MissingLayers]). +type Confidence string + +// Canonical Confidence values. The set is pinned by the v1.0-rc1 +// envelope schema's `confidence.enum`; adding a value requires a new +// SDK version. +const ( + ConfidenceFull Confidence = "full" + ConfidencePartial Confidence = "partial" +) + +// EvidenceRef is one entry in a Verdict's evidence trail. Each entry +// names which receiver-side surface the evidence came from, its +// upstream UID, the wall-clock timestamp, and a one-line human-readable +// summary. The trail reads top-to-bottom as the operator-facing +// timeline. +type EvidenceRef struct { + // Kind names the receiver-side surface the evidence came from. + // v1.0-rc1 vocabulary: pod_event, node_condition, nccl_fr, + // kernel_event, hw_error, hw_throttle, hw_io_collapse, hw_fb, + // cuda_oom, pcie_aer, ib_port_state. + Kind string `json:"kind"` + + // UID is the upstream identifier — for Kubernetes Events it's the + // Event.UID; for node conditions a tracecore-synthetic + // `{node-uid}/{pressure-kind}/{transition-unix-nano}`. + UID string `json:"uid"` + + // Timestamp is the evidence's wall-clock time (RFC 3339). + Timestamp time.Time `json:"timestamp"` + + // Description is the one-line operator-facing summary. + Description string `json:"description"` +} + +// Verdict is the typed v1.0-rc1 envelope. The four required fields +// (PatternID, Headline, Remediation, EvidenceTrail) are the contract +// every consumer can rely on. Confidence + MissingLayers are +// envelope-optional. Per-pattern extensions live in Extras. +type Verdict struct { + // PatternID is the string-typed numeric pattern identifier (e.g. + // "14" for pod_evicted). String-typed so future namespacing + // (e.g. "vendor.42") does not break the wire format. + PatternID string `json:"pattern.id"` + + // Headline is the one-line operator-facing summary. Suitable as an + // alert title. + Headline string `json:"headline"` + + // Remediation is the detector-controlled remediation hint. + // Suitable as alert body text. + Remediation string `json:"remediation"` + + // Confidence is the join-completeness signal. Omitted for patterns + // whose emission rule is "all layers joined or no verdict". + Confidence Confidence `json:"confidence,omitempty"` + + // EvidenceTrail is the ordered list of evidence references that + // drove the verdict. Always populated; schema enforces minItems=1. + EvidenceTrail []EvidenceRef `json:"evidence_trail"` + + // MissingLayers names evidence layers that did not join, in causal + // order. Empty / omitted when Confidence==Full or when the pattern + // does not carry Confidence. + MissingLayers []string `json:"missing_layers,omitempty"` + + // Extras carries every JSON property the envelope schema did not + // bind to a typed field above — the envelope sets + // `additionalProperties: true` so per-pattern extensions + // (k8s.pod.name, gpu_id, xid_code, kind, …) survive the decode. + // Consumers reading pattern-specific fields type-assert through + // Extras; see Decode's package-level example. + Extras map[string]any `json:"-"` +} From de3e8f62ce165ec6dfa2dacea129c4112b569356 Mon Sep 17 00:00:00 2001 From: Tri Lam Date: Mon, 1 Jun 2026 02:24:26 -0700 Subject: [PATCH 5/9] =?UTF-8?q?test(tracecore=5Fverdict):=20RED=20?= =?UTF-8?q?=E2=80=94=20Python=20verdict-consumption=20SDK=20decoder?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirrors the Go SDK's TDD discipline: every envelope guard + additionalProperties+ shipped-pattern fixture is encoded here as a pytest row. Fails with ModuleNotFoundError until __init__.py + verdict.py + decode.py land. Drives criterion 12 (Python half). Signed-off-by: Tri Lam --- python/tracecore_verdict/test_decode.py | 363 ++++++++++++++++++++++++ 1 file changed, 363 insertions(+) create mode 100644 python/tracecore_verdict/test_decode.py diff --git a/python/tracecore_verdict/test_decode.py b/python/tracecore_verdict/test_decode.py new file mode 100644 index 00000000..053fa7c2 --- /dev/null +++ b/python/tracecore_verdict/test_decode.py @@ -0,0 +1,363 @@ +# SPDX-License-Identifier: Apache-2.0 +"""pytest suite for the v1.0-rc1 Verdict-consumption SDK. + +The tests double as the executable contract for criterion 12 (Python +half). Adding a pattern verdict at this rev MUST add a row to +``test_decode_all_shipped_patterns_round_trip``. +""" + +from __future__ import annotations + +import json +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +from tracecore_verdict import Confidence, EvidenceRef, Verdict, decode +from tracecore_verdict.decode import VerdictDecodeError + + +GOOD_VERDICT_JSON = json.dumps( + { + "pattern.id": "14", + "headline": "Pod ns/p evicted at 2026-05-18T10:00:00Z due to disk pressure", + "remediation": "relocate workload to a node with NVMe-backed ephemeral storage", + "confidence": "full", + "evidence_trail": [ + { + "kind": "pod_event", + "uid": "evt-1", + "timestamp": "2026-05-18T10:00:00Z", + "description": "Evicted: disk pressure", + }, + { + "kind": "node_condition", + "uid": "node-a/disk/1716030000000000000", + "timestamp": "2026-05-18T09:59:55Z", + "description": "DiskPressure transitioned true", + }, + ], + } +) + + +def test_decode_happy_path() -> None: + """A known-good v1.0-rc1 envelope decodes into a typed Verdict.""" + v = decode(GOOD_VERDICT_JSON) + assert isinstance(v, Verdict) + assert v.pattern_id == "14" + assert "Pod ns/p evicted" in v.headline + assert "NVMe" in v.remediation + assert v.confidence is Confidence.FULL + assert len(v.evidence_trail) == 2 + assert v.evidence_trail[0].kind == "pod_event" + assert v.evidence_trail[0].uid == "evt-1" + assert v.evidence_trail[0].timestamp == datetime( + 2026, 5, 18, 10, 0, 0, tzinfo=timezone.utc + ) + assert v.evidence_trail[0].description == "Evicted: disk pressure" + assert v.evidence_trail[1].kind == "node_condition" + + +def test_decode_accepts_bytes_input() -> None: + """decode() accepts bytes as well as str.""" + v = decode(GOOD_VERDICT_JSON.encode("utf-8")) + assert v.pattern_id == "14" + + +@pytest.mark.parametrize( + "label,payload", + [ + ( + "missing_pattern_id", + { + "headline": "h", + "remediation": "r", + "evidence_trail": [ + { + "kind": "pod_event", + "uid": "u", + "timestamp": "2026-05-18T10:00:00Z", + "description": "d", + } + ], + }, + ), + ( + "missing_headline", + { + "pattern.id": "14", + "remediation": "r", + "evidence_trail": [ + { + "kind": "pod_event", + "uid": "u", + "timestamp": "2026-05-18T10:00:00Z", + "description": "d", + } + ], + }, + ), + ( + "empty_evidence_trail", + { + "pattern.id": "14", + "headline": "h", + "remediation": "r", + "evidence_trail": [], + }, + ), + ( + "pattern_id_numeric_not_string", + { + "pattern.id": 14, + "headline": "h", + "remediation": "r", + "evidence_trail": [ + { + "kind": "pod_event", + "uid": "u", + "timestamp": "2026-05-18T10:00:00Z", + "description": "d", + } + ], + }, + ), + ( + "confidence_outside_enum", + { + "pattern.id": "14", + "headline": "h", + "remediation": "r", + "confidence": "maybe", + "evidence_trail": [ + { + "kind": "pod_event", + "uid": "u", + "timestamp": "2026-05-18T10:00:00Z", + "description": "d", + } + ], + }, + ), + ( + "evidence_uid_empty", + { + "pattern.id": "14", + "headline": "h", + "remediation": "r", + "evidence_trail": [ + { + "kind": "pod_event", + "uid": "", + "timestamp": "2026-05-18T10:00:00Z", + "description": "d", + } + ], + }, + ), + ], +) +def test_decode_rejects_schema_invalid(label: str, payload: dict) -> None: + """Every envelope-level guard surfaces as VerdictDecodeError.""" + with pytest.raises(VerdictDecodeError): + decode(json.dumps(payload)) + + +def test_decode_rejects_malformed_json() -> None: + """Non-JSON input surfaces as VerdictDecodeError, not JSONDecodeError.""" + with pytest.raises(VerdictDecodeError): + decode("{not json") + + +def test_decode_preserves_extras() -> None: + """Per-pattern extensions ride through on Verdict.extras.""" + payload = { + "pattern.id": "14", + "headline": "h", + "remediation": "r", + "confidence": "full", + "evidence_trail": [ + { + "kind": "pod_event", + "uid": "u", + "timestamp": "2026-05-18T10:00:00Z", + "description": "d", + } + ], + "k8s.pod.name": "trainer-3", + "k8s.pod.namespace": "ml-prod", + "k8s.node.name": "node-a", + "some_future_field": {"nested": [1, 2, 3]}, + } + v = decode(json.dumps(payload)) + assert v.extras["k8s.pod.name"] == "trainer-3" + assert v.extras["k8s.pod.namespace"] == "ml-prod" + assert v.extras["k8s.node.name"] == "node-a" + assert v.extras["some_future_field"] == {"nested": [1, 2, 3]} + + # Typed envelope fields don't duplicate into extras. + assert "headline" not in v.extras + assert "pattern.id" not in v.extras + assert "evidence_trail" not in v.extras + + +def _ev(kind: str) -> dict: + return { + "kind": kind, + "uid": "u", + "timestamp": "2026-05-18T10:00:00Z", + "description": "d", + } + + +# Mirror of the Go SDK's TestDecode_AllShippedPatternsRoundTrip: every +# pattern verdict shipped at v1.0-rc1 must decode cleanly. Adding a +# pattern in a future PR MUST add a row here. +SHIPPED_PATTERN_FIXTURES = { + "pod_evicted": { + "pattern.id": "14", + "headline": "h", + "remediation": "r", + "confidence": "full", + "evidence_trail": [_ev("pod_event"), _ev("node_condition")], + "k8s.pod.name": "p", + "k8s.pod.namespace": "ns", + "k8s.node.name": "n", + }, + "nccl_hang": { + "pattern.id": "15", + "headline": "h", + "remediation": "r", + "evidence_trail": [_ev("nccl_fr")], + "pg_id": 7, + "collective_seq_id": 42, + "hanging_ranks": [1, 3], + }, + "xid_correlation": { + "pattern.id": "16", + "headline": "h", + "remediation": "r", + "evidence_trail": [_ev("kernel_event"), _ev("pod_event")], + "xid_code": 79, + "node": "n", + "evicted_pod": "ns/p", + }, + "hbm_ecc": { + "pattern.id": "17", + "headline": "h", + "remediation": "r", + "evidence_trail": [_ev("hw_error"), _ev("kernel_event")], + "xid_code": 48, + "gpu_id": "0000:31:00.0", + "ecc_delta": 1, + "node": "n", + }, + "thermal_throttle": { + "pattern.id": "18", + "headline": "h", + "remediation": "r", + "evidence_trail": [_ev("hw_throttle"), _ev("hw_throttle")], + "node": "n", + "gpu_count": 2, + "gpu_ids": ["0000:31:00.0", "0000:32:00.0"], + }, + "pcie_aer": { + "pattern.id": "19", + "headline": "h", + "remediation": "r", + "evidence_trail": [_ev("pcie_aer"), _ev("hw_io_collapse")], + "gpu_id": "0000:31:00.0", + "severity": "Fatal", + "aer_type": "Data Link Layer", + "drop_ratio": 0.9, + "node": "n", + }, + "cuda_oom": { + "pattern.id": "20", + "headline": "h", + "remediation": "r", + "confidence": "full", + "evidence_trail": [_ev("cuda_oom"), _ev("hw_fb")], + "gpu_id": "0000:31:00.0", + "node": "n", + "kind": "true_oom", + "tried_alloc_bytes": 1024, + "fb_free_bytes": 0, + "fb_free_ratio": 0.0, + }, + "ib_link_flap": { + "pattern.id": "21", + "headline": "h", + "remediation": "r", + "confidence": "partial", + "evidence_trail": [_ev("ib_port_state")], + "missing_layers": ["hw_throttle"], + "node": "n", + "hca_device": "mlx5_0", + "port": 1, + "transition_count": 3, + }, +} + + +@pytest.mark.parametrize("name,payload", list(SHIPPED_PATTERN_FIXTURES.items())) +def test_decode_all_shipped_patterns_round_trip(name: str, payload: dict) -> None: + """All 8 shipped pattern verdicts decode cleanly.""" + v = decode(json.dumps(payload)) + assert v.pattern_id == payload["pattern.id"] + assert len(v.evidence_trail) >= 1 + + +def test_decode_missing_layers_populated() -> None: + """missing_layers surfaces on the typed struct.""" + payload = { + "pattern.id": "14", + "headline": "h", + "remediation": "r", + "confidence": "partial", + "evidence_trail": [ + { + "kind": "pod_event", + "uid": "u", + "timestamp": "2026-05-18T10:00:00Z", + "description": "d", + } + ], + "missing_layers": ["node_condition"], + } + v = decode(json.dumps(payload)) + assert v.confidence is Confidence.PARTIAL + assert v.missing_layers == ["node_condition"] + + +def test_evidence_ref_dataclass() -> None: + """EvidenceRef carries the four envelope-required fields.""" + ref = EvidenceRef( + kind="pod_event", + uid="u", + timestamp=datetime(2026, 5, 18, 10, 0, 0, tzinfo=timezone.utc), + description="d", + ) + assert ref.kind == "pod_event" + + +def test_embedded_schema_matches_canonical() -> None: + """The package-data schema is byte-identical to docs/schemas/... + + The Python build copies the canonical schema in alongside the + package (single source of truth); drift fails the build at this + test. + """ + pkg_dir = Path(__file__).parent + embedded = (pkg_dir / "schema.json").read_bytes() + canonical = ( + pkg_dir / ".." / ".." / "docs" / "schemas" / "verdict-1.0.0-rc1.json" + ).read_bytes() + assert embedded == canonical, ( + "python/tracecore_verdict/schema.json drifted from " + "docs/schemas/verdict-1.0.0-rc1.json. Run: " + "cp docs/schemas/verdict-1.0.0-rc1.json " + "python/tracecore_verdict/schema.json" + ) From 34fe772456b72065ea4be1ecece741e770405847 Mon Sep 17 00:00:00 2001 From: Tri Lam Date: Mon, 1 Jun 2026 02:26:51 -0700 Subject: [PATCH 6/9] =?UTF-8?q?feat(tracecore=5Fverdict):=20GREEN=20?= =?UTF-8?q?=E2=80=94=20Python=20verdict-consumption=20SDK?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Publishes python/tracecore_verdict — the pip-installable Python client for the v1.0-rc1 Verdict envelope. decode() validates against the package-data copy of docs/schemas/verdict-1.0.0-rc1.json; a sync test pins byte-equality with the canonical source. Per-pattern extensions ride through Verdict.extras. Drives v1-rc1 cut criterion 12 (Python half). Signed-off-by: Tri Lam --- python/tracecore_verdict/Makefile | 35 ++++++ python/tracecore_verdict/README.md | 103 ++++++++++++++++ python/tracecore_verdict/__init__.py | 34 ++++++ python/tracecore_verdict/decode.py | 154 ++++++++++++++++++++++++ python/tracecore_verdict/py.typed | 0 python/tracecore_verdict/pyproject.toml | 52 ++++++++ python/tracecore_verdict/schema.json | 83 +++++++++++++ python/tracecore_verdict/verdict.py | 91 ++++++++++++++ 8 files changed, 552 insertions(+) create mode 100644 python/tracecore_verdict/Makefile create mode 100644 python/tracecore_verdict/README.md create mode 100644 python/tracecore_verdict/__init__.py create mode 100644 python/tracecore_verdict/decode.py create mode 100644 python/tracecore_verdict/py.typed create mode 100644 python/tracecore_verdict/pyproject.toml create mode 100644 python/tracecore_verdict/schema.json create mode 100644 python/tracecore_verdict/verdict.py diff --git a/python/tracecore_verdict/Makefile b/python/tracecore_verdict/Makefile new file mode 100644 index 00000000..bd5fef97 --- /dev/null +++ b/python/tracecore_verdict/Makefile @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: Apache-2.0 +# Convenience entry points for the tracecore-verdict SDK. Mirrors the +# tracecore_pyspy Makefile shape. CI calls `make test` directly. + +PYTHON ?= python3 +PIP ?= $(PYTHON) -m pip + +.PHONY: help install test lint build clean sync-schema + +help: + @echo "Targets:" + @echo " install pip-install in editable mode w/ test extras" + @echo " test run pytest" + @echo " build build the sdist + wheel" + @echo " sync-schema copy the canonical schema from docs/schemas/" + @echo " clean remove build artifacts" + +install: + $(PIP) install -e '.[test]' + +test: + $(PYTHON) -m pytest . + +build: + $(PYTHON) -m pip install --upgrade build + $(PYTHON) -m build + +# Refresh the package-data schema copy from the canonical source of +# truth at docs/schemas/. The test suite includes a sync-check that +# fails if these drift. +sync-schema: + cp ../../docs/schemas/verdict-1.0.0-rc1.json schema.json + +clean: + rm -rf build dist *.egg-info .pytest_cache __pycache__ diff --git a/python/tracecore_verdict/README.md b/python/tracecore_verdict/README.md new file mode 100644 index 00000000..92122839 --- /dev/null +++ b/python/tracecore_verdict/README.md @@ -0,0 +1,103 @@ +# tracecore-verdict (Python) + +Client-side SDK for consuming **TraceCore Verdict v1.0-rc1** envelopes. + +This is the Python half of v1.0-rc1 cut-criterion 12 +("Verdict-consumption SDKs"). The Go sibling lives at +[`module/sdk/verdict/`](../../module/sdk/verdict). + +## Install + +```bash +pip install tracecore-verdict +``` + +Tag the consuming version against the `v1.0.0-rc1` release. The +package targets Python 3.8+. + +## Use + +```python +from tracecore_verdict import decode + +raw = b"""{ + "pattern.id": "14", + "headline": "Pod ml-prod/trainer-3 evicted at 2026-05-18T10:00:00Z due to disk pressure", + "remediation": "relocate workload to a node with NVMe-backed ephemeral storage", + "confidence": "full", + "evidence_trail": [ + {"kind": "pod_event", "uid": "evt-1", + "timestamp": "2026-05-18T10:00:00Z", + "description": "Evicted: disk pressure"} + ], + "k8s.pod.name": "trainer-3" +}""" + +v = decode(raw) +print(v.pattern_id, v.headline) # 14 Pod ml-prod/trainer-3 evicted ... +print(v.confidence.value) # "full" +print(v.extras["k8s.pod.name"]) # trainer-3 +``` + +## What's in the box + +| Symbol | Purpose | +| --- | --- | +| `Verdict` (dataclass) | `pattern_id`, `headline`, `remediation`, `confidence`, `evidence_trail`, `missing_layers`, `extras` | +| `EvidenceRef` (dataclass) | One row in `evidence_trail` (`kind`, `uid`, `timestamp`, `description`) | +| `Confidence` (Enum) | `FULL` \| `PARTIAL` | +| `decode(raw_json: bytes | str) -> Verdict` | JSON → schema-validated typed Verdict | +| `VerdictDecodeError` | Raised on malformed JSON or envelope-schema failure | + +## Pattern-specific fields + +The envelope sets `additionalProperties: true`, so every per-pattern +extension (`gpu_id`, `xid_code`, `kind`, `k8s.pod.name`, …) rides +through on `Verdict.extras` (`dict[str, Any]`): + +```python +pod = v.extras.get("k8s.pod.name") # pod_evicted +xid = v.extras.get("xid_code") # xid_correlation / hbm_ecc +severity = v.extras.get("severity") # pcie_aer +``` + +## Versioning + +This package targets **schema v1.0-rc1 only**. + +A future envelope revision lands as a new package (e.g. +`tracecore-verdict-v2`) — never a silent breaking change here. The +SDK's `schema.json` is byte-pinned against +`docs/schemas/verdict-1.0.0-rc1.json` by +`test_embedded_schema_matches_canonical`; drift fails the build. + +## Errors + +`decode()` raises `VerdictDecodeError` for: + +- malformed JSON / non-UTF-8 input +- missing envelope-required field (`pattern.id`, `headline`, + `remediation`, `evidence_trail`) +- `confidence` outside the `full|partial` enum +- empty `evidence_trail` +- evidence-row missing `kind` / `uid` / `timestamp` / `description` +- non-RFC3339 evidence timestamps +- any other v1.0-rc1 envelope violation + +Per-pattern field constraints (e.g. `pcie_aer.severity ∈ {Fatal, +Non-Fatal, Corrected}`) are out of scope — those live on per-pattern +schemas under `module/pkg/patterns/testdata/`. + +## Develop + +```bash +cd python/tracecore_verdict +pip install -e '.[test]' +pytest +# or +make test +``` + +## License + +Apache-2.0. diff --git a/python/tracecore_verdict/__init__.py b/python/tracecore_verdict/__init__.py new file mode 100644 index 00000000..e079da31 --- /dev/null +++ b/python/tracecore_verdict/__init__.py @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: Apache-2.0 +"""TraceCore Verdict-consumption SDK (Python). + +Client-side SDK for consuming TraceCore Verdict v1.0-rc1 envelopes. +The Go sibling lives at ``module/sdk/verdict/`` in this repo. + +Quick start:: + + from tracecore_verdict import decode + + raw = b'{"pattern.id": "14", "headline": "...", ...}' + v = decode(raw) + print(v.pattern_id, v.headline) + pod = v.extras.get("k8s.pod.name") # per-pattern extension + +This package targets schema v1.0-rc1 only. A future envelope revision +ships as a new sibling package (e.g. ``tracecore-verdict-v2``). +""" + +from __future__ import annotations + +from .decode import VerdictDecodeError, decode +from .verdict import Confidence, EvidenceRef, Verdict + + +__all__ = [ + "Confidence", + "EvidenceRef", + "Verdict", + "VerdictDecodeError", + "decode", +] + +__version__ = "1.0.0rc1" diff --git a/python/tracecore_verdict/decode.py b/python/tracecore_verdict/decode.py new file mode 100644 index 00000000..7983a6cf --- /dev/null +++ b/python/tracecore_verdict/decode.py @@ -0,0 +1,154 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Schema-validated Verdict decoder for v1.0-rc1. + +``decode(raw_json)`` returns a typed :class:`~tracecore_verdict.Verdict` +or raises :class:`VerdictDecodeError`. Per-pattern fields ride through +on :attr:`Verdict.extras`. + +The schema lives package-data alongside this module (a byte-identical +copy of ``docs/schemas/verdict-1.0.0-rc1.json``); the schema-sync test +in ``test_decode.py`` pins drift. +""" + +from __future__ import annotations + +import json +from datetime import datetime +from functools import lru_cache +from pathlib import Path +from typing import Any, Dict, Union + +from jsonschema import Draft202012Validator +from jsonschema.exceptions import ValidationError + +from .verdict import Confidence, EvidenceRef, Verdict + + +__all__ = ["decode", "VerdictDecodeError"] + + +# Envelope-level keys bound to typed Verdict fields. ``decode`` strips +# these from the raw object before populating ``Verdict.extras`` — so +# extras carry only per-pattern extensions, not duplicate copies of +# the typed envelope. +_TYPED_KEYS = { + "pattern.id", + "headline", + "remediation", + "confidence", + "evidence_trail", + "missing_layers", +} + + +class VerdictDecodeError(ValueError): + """Raised when raw input cannot be decoded into a Verdict. + + This includes malformed JSON, schema-validation failures, and + structural surprises that pass schema but break the typed + surface (e.g. evidence row with a non-RFC3339 timestamp). The + SDK's contract: ``decode()`` either returns a fully-formed + :class:`Verdict` or raises this error — never a half-populated + object. + """ + + +@lru_cache(maxsize=1) +def _validator() -> Draft202012Validator: + """Compile and memoize the v1.0-rc1 envelope schema validator.""" + schema_path = Path(__file__).parent / "schema.json" + with schema_path.open("rb") as fp: + schema = json.load(fp) + Draft202012Validator.check_schema(schema) + return Draft202012Validator(schema) + + +def _parse_timestamp(raw: str) -> datetime: + """RFC 3339 -> datetime. The envelope schema's ``format: date-time`` + plus ``Draft202012Validator`` does NOT enforce format by default + (the spec opts into format-assertion), so the typed-decode layer + parses + raises on its own.""" + # Python <3.11 fromisoformat does not accept the trailing 'Z'; + # the swap to '+00:00' is the canonical workaround. + normalized = raw.replace("Z", "+00:00") if raw.endswith("Z") else raw + try: + return datetime.fromisoformat(normalized) + except (TypeError, ValueError) as exc: + raise VerdictDecodeError( + f"verdict-sdk: evidence_trail timestamp {raw!r} is not RFC 3339" + ) from exc + + +def decode(raw_json: Union[bytes, str]) -> Verdict: + """Decode and validate a v1.0-rc1 Verdict envelope. + + Args: + raw_json: The envelope payload as bytes or str. Typically the + value of the ``pattern.verdict_json`` attribute on an OTLP + log emitted by tracecore. + + Returns: + A typed :class:`Verdict`. Per-pattern fields are accessible + via the ``extras`` dict. + + Raises: + VerdictDecodeError: malformed JSON, schema validation + failure, or a structural surprise the typed layer cannot + reconcile. + """ + if isinstance(raw_json, bytes): + try: + text = raw_json.decode("utf-8") + except UnicodeDecodeError as exc: + raise VerdictDecodeError( + "verdict-sdk: raw_json is not valid UTF-8" + ) from exc + else: + text = raw_json + + try: + payload: Any = json.loads(text) + except json.JSONDecodeError as exc: + raise VerdictDecodeError( + f"verdict-sdk: malformed JSON: {exc.msg}" + ) from exc + + try: + _validator().validate(payload) + except ValidationError as exc: + raise VerdictDecodeError( + f"verdict-sdk: schema validation failed: {exc.message}" + ) from exc + + if not isinstance(payload, dict): + # Should be unreachable — schema requires type:object — but + # belt-and-suspenders so the typed return is honest. + raise VerdictDecodeError("verdict-sdk: envelope root is not an object") + + evidence_trail = [ + EvidenceRef( + kind=row["kind"], + uid=row["uid"], + timestamp=_parse_timestamp(row["timestamp"]), + description=row["description"], + ) + for row in payload["evidence_trail"] + ] + + confidence: Confidence | None = None + if "confidence" in payload: + confidence = Confidence(payload["confidence"]) + + extras: Dict[str, Any] = { + k: v for k, v in payload.items() if k not in _TYPED_KEYS + } + + return Verdict( + pattern_id=payload["pattern.id"], + headline=payload["headline"], + remediation=payload["remediation"], + evidence_trail=evidence_trail, + confidence=confidence, + missing_layers=list(payload.get("missing_layers", [])), + extras=extras, + ) diff --git a/python/tracecore_verdict/py.typed b/python/tracecore_verdict/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/python/tracecore_verdict/pyproject.toml b/python/tracecore_verdict/pyproject.toml new file mode 100644 index 00000000..ef393770 --- /dev/null +++ b/python/tracecore_verdict/pyproject.toml @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: Apache-2.0 +# tracecore-verdict: client-side SDK for consuming v1.0-rc1 Verdict +# envelopes. Drives v1-rc1 cut criterion 12 (Python half). + +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "tracecore-verdict" +version = "1.0.0rc1" +description = "Client-side SDK for consuming TraceCore Verdict v1.0-rc1 envelopes." +readme = "README.md" +requires-python = ">=3.8" +license = { text = "Apache-2.0" } +authors = [{ name = "tracecore contributors" }] +keywords = ["tracecore", "opentelemetry", "verdict", "schema"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Software Development :: Libraries", + "Topic :: System :: Monitoring", +] +dependencies = [ + "jsonschema>=4.17", +] + +[project.optional-dependencies] +test = ["pytest>=7.0"] + +[project.urls] +Homepage = "https://github.com/tracecoreai/tracecore" +Issues = "https://github.com/tracecoreai/tracecore/issues" +Documentation = "https://github.com/tracecoreai/tracecore/blob/main/python/tracecore_verdict/README.md" + +[tool.setuptools] +# The package directory == this pyproject's directory; build context +# is the python/tracecore_verdict/ subtree (matches tracecore_pyspy +# convention). +packages = ["tracecore_verdict"] +package-dir = { tracecore_verdict = "." } + +[tool.setuptools.package-data] +tracecore_verdict = ["schema.json", "py.typed"] diff --git a/python/tracecore_verdict/schema.json b/python/tracecore_verdict/schema.json new file mode 100644 index 00000000..1fbaa251 --- /dev/null +++ b/python/tracecore_verdict/schema.json @@ -0,0 +1,83 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schema.tracecore.io/verdict/1.0.0-rc1.json", + "title": "TraceCore Verdict envelope v1.0-rc1", + "description": "Canonical envelope shape every TraceCore pattern detector emits. This is the v1.0-rc1 cut of the externally observable verdict contract — the shape downstream consumers (Prometheus alertmanager webhooks, Slack routers, audit pipelines) bind against. Per-pattern detectors layer pattern-specific fields on top of this envelope (e.g. `gpu_id`, `xid_code`, `kind`); per-pattern testdata schemas under module/pkg/patterns/testdata/_verdict.schema.json pin those extensions. The envelope intentionally sets `additionalProperties: true` so per-pattern extensions validate without re-listing here — drift in pattern-specific fields surfaces on the per-pattern schema, drift in envelope-level fields surfaces here. Versioning policy: docs/schemas/README.md; evolution policy: docs/DEPRECATION.md (forthcoming).", + "type": "object", + "required": ["pattern.id", "headline", "remediation", "evidence_trail"], + "additionalProperties": true, + "properties": { + "pattern.id": { + "type": "string", + "minLength": 1, + "description": "String-typed numeric identifier (e.g. \"14\" for pod_evicted, \"15\" for nccl_hang). Uniform across pattern detectors per MILESTONES.md §M19 rubric L405. String-typed (not integer) so future namespacing (e.g. \"vendor.42\") does not break the wire format." + }, + "headline": { + "type": "string", + "minLength": 1, + "description": "One-line operator-facing summary. Detector-controlled prose. Suitable as an alert title." + }, + "remediation": { + "type": "string", + "minLength": 1, + "description": "Detector-controlled remediation hint. Suitable as alert body text. Detectors key the prose on the underlying root cause (e.g. for pod_evicted: disk → relocate.*NVMe; memory → reduce per-rank memory; pid → cap fork rate)." + }, + "confidence": { + "type": "string", + "enum": ["full", "partial"], + "description": "How complete the evidence trail is. `full` = every required evidence layer joined; `partial` = at least one layer missing (recorded by name in `missing_layers`). Optional at the envelope level — patterns whose emission rule is \"both layers joined or no verdict\" (hbm_ecc, pcie_aer, thermal_throttle, xid_correlation, nccl_hang) omit this field rather than constant-stamp `full`." + }, + "evidence_trail": { + "type": "array", + "minItems": 1, + "description": "Ordered list of evidence references that drove the verdict. Reads top-to-bottom as the operator-facing timeline. Per-pattern schemas tighten `minItems` (e.g. hbm_ecc requires 2 entries — one hw_error, one kernel_event).", + "items": { + "type": "object", + "required": ["kind", "uid", "timestamp", "description"], + "additionalProperties": false, + "properties": { + "kind": { + "type": "string", + "enum": [ + "pod_event", + "node_condition", + "nccl_fr", + "kernel_event", + "hw_error", + "hw_throttle", + "hw_io_collapse", + "hw_fb", + "cuda_oom", + "pcie_aer", + "ib_port_state" + ], + "description": "Receiver-side surface the evidence came from. Per-pattern schemas tighten this to a pattern-specific subset (e.g. pod_evicted: [pod_event, node_condition]; pcie_aer: [pcie_aer, hw_io_collapse]). New kinds added here MUST land alongside a new EvidenceKind* constant in module/pkg/patterns/ and a per-pattern schema update." + }, + "uid": { + "type": "string", + "minLength": 1, + "description": "Upstream identifier. For Kubernetes Events: Event.UID; for node conditions: tracecore-synthetic `{node-uid}/{pressure-kind}/{transition-unix-nano}`; for kernel/nccl/hw events: receiver-specific stable UID." + }, + "timestamp": { + "type": "string", + "format": "date-time", + "description": "Evidence wall-clock time (RFC 3339 / ISO 8601). For k8s Events: Event.EventTime; for node conditions: LastTransitionTime." + }, + "description": { + "type": "string", + "minLength": 1, + "description": "One-line operator-facing summary. Detector-controlled prose; not parsed by anything downstream." + } + } + } + }, + "missing_layers": { + "type": "array", + "description": "Names of evidence layers that did not join, populated in causal order. Empty / omitted when `confidence` is `full` (or when the pattern does not carry `confidence`). Vocabulary is shared with `evidence_trail[].kind` — a missing layer is named by the kind it would have carried.", + "items": { + "type": "string", + "minLength": 1 + } + } + } +} diff --git a/python/tracecore_verdict/verdict.py b/python/tracecore_verdict/verdict.py new file mode 100644 index 00000000..48fafbb0 --- /dev/null +++ b/python/tracecore_verdict/verdict.py @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Typed dataclasses for the TraceCore Verdict v1.0-rc1 envelope. + +This module defines the client-facing shape; runtime decoding + +schema validation live in :mod:`tracecore_verdict.decode`. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime +from enum import Enum +from typing import Any, Dict, List, Optional + + +class Confidence(str, Enum): + """Join-completeness signal for a Verdict. + + ``FULL`` means every required evidence layer joined; ``PARTIAL`` + means at least one was missing (named in + :attr:`Verdict.missing_layers`). The envelope keeps this field + optional — patterns whose emission rule is "all layers or no + verdict" omit it rather than constant-stamp ``FULL``. + """ + + FULL = "full" + PARTIAL = "partial" + + +@dataclass(frozen=True) +class EvidenceRef: + """One row in a Verdict's evidence trail. + + Trail rows read top-to-bottom as the operator-facing timeline. + """ + + kind: str + """Receiver-side surface. v1.0-rc1 vocabulary: ``pod_event``, + ``node_condition``, ``nccl_fr``, ``kernel_event``, ``hw_error``, + ``hw_throttle``, ``hw_io_collapse``, ``hw_fb``, ``cuda_oom``, + ``pcie_aer``, ``ib_port_state``.""" + + uid: str + """Upstream identifier — Kubernetes Event UID, synthetic node- + condition key, or receiver-specific stable UID.""" + + timestamp: datetime + """Wall-clock time (RFC 3339).""" + + description: str + """One-line operator-facing summary.""" + + +@dataclass +class Verdict: + """Typed v1.0-rc1 Verdict envelope. + + The four envelope-required fields (``pattern_id``, ``headline``, + ``remediation``, ``evidence_trail``) are the load-bearing contract + every consumer can rely on. ``confidence`` + ``missing_layers`` + are envelope-optional. Per-pattern extensions live on + :attr:`extras` (the envelope sets ``additionalProperties: true``). + """ + + pattern_id: str + """String-typed numeric pattern identifier (e.g. ``"14"``). JSON + wire key: ``pattern.id``.""" + + headline: str + """One-line operator-facing summary. Suitable as an alert title.""" + + remediation: str + """Detector-controlled remediation hint. Suitable as alert body.""" + + evidence_trail: List[EvidenceRef] + """Ordered evidence trail. Schema enforces ``minItems=1``.""" + + confidence: Optional[Confidence] = None + """Join-completeness signal. ``None`` for patterns whose emission + rule is "all layers joined or no verdict".""" + + missing_layers: List[str] = field(default_factory=list) + """Names of evidence layers that did not join, in causal order. + Empty when ``confidence is Confidence.FULL`` or when the pattern + does not carry confidence.""" + + extras: Dict[str, Any] = field(default_factory=dict) + """Every JSON property the envelope did not bind to a typed field + above. Per-pattern extensions (``k8s.pod.name``, ``gpu_id``, + ``xid_code``, ``kind``, …) ride through here unchanged. Consumers + reading pattern-specific fields look them up by key.""" From 59eda7a072d364711d3a897b6c99b6074d9cf808 Mon Sep 17 00:00:00 2001 From: Tri Lam Date: Mon, 1 Jun 2026 02:31:02 -0700 Subject: [PATCH 7/9] ci+docs(sdk): gate Python SDK + flip v1-rc1 criterion 12 Adds sdk-python CI job (pytest python/tracecore_verdict) wired into the verify aggregator. Go SDK is already covered by verify-test via go.work resolution. Publishes docs/sdk/README.md as the SDK index + cross-links it from docs/README.md. Flips cut criterion 12 to shipped. Signed-off-by: Tri Lam --- .github/workflows/ci.yml | 28 ++++++++- docs/README.md | 1 + docs/sdk/README.md | 117 ++++++++++++++++++++++++++++++++++++ docs/v1-rc1-cut-criteria.md | 13 +++- 4 files changed, 157 insertions(+), 2 deletions(-) create mode 100644 docs/sdk/README.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6cc71777..d7832ede 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -155,10 +155,32 @@ jobs: - name: smoke-quickstart run: make smoke-quickstart + # v1-rc1 cut criterion 12: Python verdict-consumption SDK. + # The Go half (module/sdk/verdict/) is already covered by verify-test + # because `go test -race ./...` resolves it via go.work; the Python + # half needs its own minimal job. Lightweight on purpose — single + # Python version + pytest run, no matrix. If the SDK grows, this is + # where a Python-version matrix lands. + sdk-python: + name: sdk-python + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version: "3.11" + cache: "pip" + - name: install + run: | + python -m pip install --upgrade pip + pip install -e 'python/tracecore_verdict[test]' + - name: pytest + run: pytest python/tracecore_verdict/ + verify: name: verify runs-on: ubuntu-latest - needs: [verify-test, verify-lint, verify-static, validator-recipe] + needs: [verify-test, verify-lint, verify-static, validator-recipe, sdk-python] if: always() steps: - name: aggregator @@ -180,6 +202,10 @@ jobs: echo "::error::validator-recipe did not succeed (result=${{ needs.validator-recipe.result }})" fail=1 fi + if [ "${{ needs.sdk-python.result }}" != "success" ]; then + echo "::error::sdk-python did not succeed (result=${{ needs.sdk-python.result }})" + fail=1 + fi if [ "$fail" -eq 1 ]; then echo "one or more verify-* sub-jobs did not succeed; failing aggregator" exit 1 diff --git a/docs/README.md b/docs/README.md index b6295acc..1fccaa29 100644 --- a/docs/README.md +++ b/docs/README.md @@ -39,6 +39,7 @@ Legend: 👤 operator · 🛠️ contributor · 🏛️ maintainer · 🌐 exter | [proposals/](proposals/) | 🏛️ | Drafts pending upstream (semconv extensions, etc.). | | [research/](research/) | 🛠️ | Synthesized findings from reading external sources (OTel collector internals, benchmark baselines). | | [schemas/](schemas/) | 🛠️ | Receiver schema documents pointed at by emitted `SchemaURL`. | +| [sdk/](sdk/README.md) | 🌐 👤 | Verdict-consumption SDKs (Python + Go) — typed clients for the v1.0-rc1 envelope. Closes v1-rc1 cut criterion 12. | | [examples/](examples/) | 👤 | Reference operator artifacts (Prometheus alerts, Grafana dashboard, with-telemetry config). | | [followups/](followups/) | 🏛️ | Per-milestone follow-up shards + cross-cutting `_needs-prod-data` / `_needs-gpu` buckets. See [followups/README.md](followups/README.md) for filing convention. | | [integrations/](integrations/) | 👤 | Validated recipes for shipping tracecore output to specific backends. See per-recipe rows below. | diff --git a/docs/sdk/README.md b/docs/sdk/README.md new file mode 100644 index 00000000..e7977757 --- /dev/null +++ b/docs/sdk/README.md @@ -0,0 +1,117 @@ +# Verdict-consumption SDKs + +Typed client SDKs for consuming **TraceCore Verdict v1.0-rc1** +envelopes. These ship alongside every release tagged at the matching +schema version (e.g. `v1.0.0-rc1`) and close +[v1-rc1 cut-criterion 12](../v1-rc1-cut-criteria.md#12-verdict-consumption-sdks-python--go). + +## Which one do I pick? + +| If you are… | Use | +| --- | --- | +| Writing a Prometheus alertmanager webhook in Python, or routing verdicts into Slack via FastAPI | [`tracecore-verdict`](../../python/tracecore_verdict/) (Python) | +| Writing an OTel-collector processor, an exporter, or any downstream Go consumer | [`module/sdk/verdict`](../../module/sdk/verdict/) (Go) | +| Calling from another language | Validate against the published JSON Schema at `https://schema.tracecore.io/verdict/1.0.0-rc1.json` (mirrored in [`docs/schemas/verdict-1.0.0-rc1.json`](../schemas/verdict-1.0.0-rc1.json)) | + +Both SDKs: + +- **Target schema v1.0-rc1 only.** A future envelope ships a new + sibling package — never a silent breaking change. +- **Validate before they hand back a typed object.** A schema-invalid + payload surfaces as an error, not a half-populated struct. +- **Preserve per-pattern extensions** (`gpu_id`, `xid_code`, `kind`, + `k8s.pod.name`, …) via an `Extras` / `extras` map. The envelope + sets `additionalProperties: true`; the SDK rides that through + unchanged. +- **Pin a byte-identical copy of `docs/schemas/verdict-1.0.0-rc1.json`** + as the source of truth. CI's drift test fails the build if the SDK + copy strays from the canonical. + +## What's in the envelope + +| Field | Required | Notes | +| --- | --- | --- | +| `pattern.id` | yes | String-typed numeric (e.g. `"14"`). Future namespacing (`"vendor.42"`) keeps the wire-format stable. | +| `headline` | yes | One-line operator-facing summary. Alert title. | +| `remediation` | yes | Detector-controlled remediation hint. Alert body. | +| `evidence_trail` | yes | Ordered list; `minItems: 1`. Reads top-to-bottom as the timeline. | +| `confidence` | no | `full` \| `partial`. Patterns whose emission rule is "all layers or no verdict" omit it. | +| `missing_layers` | no | Names of evidence layers that did not join. Empty when `confidence == full`. | +| `` | no | Anything else — `k8s.pod.name`, `gpu_id`, `xid_code`, `kind`, … — rides through on Extras. | + +## Install + +```bash +# Python +pip install tracecore-verdict + +# Go +go get github.com/tracecoreai/tracecore/module/sdk/verdict +``` + +## Quick start (Python) + +```python +from tracecore_verdict import decode + +v = decode(raw_bytes) +print(v.pattern_id, v.headline) +pod = v.extras.get("k8s.pod.name") # per-pattern extension +``` + +## Quick start (Go) + +```go +import "github.com/tracecoreai/tracecore/module/sdk/verdict" + +v, err := verdict.Decode(rawJSON) +if err != nil { return err } +fmt.Println(v.PatternID, v.Headline) +if pod, ok := v.Extras["k8s.pod.name"].(string); ok { /* … */ } +``` + +## Patterns shipped at v1.0-rc1 + +The envelope is the **union-superset** of every pattern's emitted +shape. The eight verdicts shipped at v1.0-rc1: + +| ID | Name | Per-pattern extensions | +| --- | --- | --- | +| 14 | `pod_evicted` | `k8s.pod.name`, `k8s.pod.namespace`, `k8s.node.name`, `k8s.event.reason` | +| 15 | `nccl_hang` | `pg_id`, `collective_seq_id`, `hanging_ranks` | +| 16 | `xid_correlation` | `xid_code`, `node`, `evicted_pod` | +| 17 | `hbm_ecc` | `xid_code`, `gpu_id`, `ecc_delta`, `node` | +| 18 | `thermal_throttle` | `node`, `gpu_count`, `gpu_ids` | +| 19 | `pcie_aer` | `gpu_id`, `severity`, `aer_type`, `drop_ratio`, `node` | +| 20 | `cuda_oom` | `gpu_id`, `node`, `kind`, `tried_alloc_bytes`, `fb_free_bytes`, `fb_free_ratio` | +| 21 | `ib_link_flap` | `node`, `hca_device`, `port`, `transition_count` | + +Each SDK ships a parametrized test that round-trips one fixture per +row above. Adding a pattern in a future PR MUST extend that table. + +## Versioning policy + +- **`v1.0.0-rc1`** is the first tagged schema version. Both SDKs key + off the schema's `$id` + (`https://schema.tracecore.io/verdict/1.0.0-rc1.json`). +- A backwards-compatible field-add (still rides on `additionalProperties`) + is a **patch** bump at the schema level; SDKs that bind only typed + fields keep working without an upgrade. +- A breaking change (new required field, type change, narrowed enum) + is a **major** bump and lands as a new sibling SDK package + (`tracecore-verdict-v2` / `module/sdk/verdict/v2`). + +See [`docs/schemas/README.md`](../schemas/README.md) for the schema +evolution policy and [`docs/DEPRECATION.md`](../DEPRECATION.md) for +the rollout cadence. + +## Source layout + +``` +docs/schemas/verdict-1.0.0-rc1.json # canonical source of truth +module/sdk/verdict/ # Go SDK (embeds schema.json) +python/tracecore_verdict/ # Python SDK (package-data schema.json) +``` + +Both SDKs include a sync-check test that fails the build if either +copy drifts from the canonical. diff --git a/docs/v1-rc1-cut-criteria.md b/docs/v1-rc1-cut-criteria.md index 30d4723c..fcc145d6 100644 --- a/docs/v1-rc1-cut-criteria.md +++ b/docs/v1-rc1-cut-criteria.md @@ -231,7 +231,18 @@ Status is set against the repo state as of 2026-05-31. RC gate "Verdict-consumption SDKs tagged at RC schema version" and [`NORTHSTARS.md` §O4 hero KPI](NORTHSTARS.md#o4-standards) (external `gen_ai.training.*` implementations). -- **Status:** ☐ — neither SDK exists yet. +- **Status:** ☑ — both SDKs shipped. Go: [`module/sdk/verdict/`](../module/sdk/verdict); + Python: [`python/tracecore_verdict/`](../python/tracecore_verdict). + Index + version-policy: [`docs/sdk/README.md`](sdk/README.md). Each + SDK embeds a byte-identical copy of + [`docs/schemas/verdict-1.0.0-rc1.json`](schemas/verdict-1.0.0-rc1.json) + pinned by a drift test; `Decode` / `decode` validate before + returning a typed `Verdict`; per-pattern extensions ride through + on `Extras` / `extras` because the envelope sets + `additionalProperties: true`; both ship a parametrized test that + round-trips all 8 v1.0-rc1 patterns. CI: Go SDK runs through + `verify-test`; Python SDK has its own `sdk-python` job wired into + the `verify` aggregator. --- From 93c18a8600a6d0c3fd02bb0e4ad3a5292bac90e0 Mon Sep 17 00:00:00 2001 From: Tri Lam Date: Mon, 1 Jun 2026 02:41:29 -0700 Subject: [PATCH 8/9] fix(sdk): add SDC evidence kinds + JSON-normalize drift comparison Signed-off-by: Tri Lam --- docs/schemas/verdict-1.0.0-rc1.json | 4 +++- docs/sdk/README.md | 3 ++- module/sdk/verdict/decode_test.go | 13 ++++++++++ module/sdk/verdict/schema.json | 4 +++- module/sdk/verdict/schema_sync_test.go | 18 +++++++++++--- module/sdk/verdict/verdict.go | 3 ++- python/tracecore_verdict/schema.json | 4 +++- python/tracecore_verdict/test_decode.py | 32 ++++++++++++++++++++----- 8 files changed, 67 insertions(+), 14 deletions(-) diff --git a/docs/schemas/verdict-1.0.0-rc1.json b/docs/schemas/verdict-1.0.0-rc1.json index 1fbaa251..0711add3 100644 --- a/docs/schemas/verdict-1.0.0-rc1.json +++ b/docs/schemas/verdict-1.0.0-rc1.json @@ -49,7 +49,9 @@ "hw_fb", "cuda_oom", "pcie_aer", - "ib_port_state" + "ib_port_state", + "hw_gpu_sdc", + "gen_ai_training_eval_accuracy" ], "description": "Receiver-side surface the evidence came from. Per-pattern schemas tighten this to a pattern-specific subset (e.g. pod_evicted: [pod_event, node_condition]; pcie_aer: [pcie_aer, hw_io_collapse]). New kinds added here MUST land alongside a new EvidenceKind* constant in module/pkg/patterns/ and a per-pattern schema update." }, diff --git a/docs/sdk/README.md b/docs/sdk/README.md index e7977757..840aa477 100644 --- a/docs/sdk/README.md +++ b/docs/sdk/README.md @@ -73,10 +73,11 @@ if pod, ok := v.Extras["k8s.pod.name"].(string); ok { /* … */ } ## Patterns shipped at v1.0-rc1 The envelope is the **union-superset** of every pattern's emitted -shape. The eight verdicts shipped at v1.0-rc1: +shape. The nine verdicts shipped at v1.0-rc1: | ID | Name | Per-pattern extensions | | --- | --- | --- | +| 13 | `silent_data_corruption` | `kind`, `gen_ai.training.job_id`, `accuracy_drop`, `baseline_accuracy`, `observed_accuracy`, `suspect_gpu_id`, `suspect_node`, `sdc_counter_delta` | | 14 | `pod_evicted` | `k8s.pod.name`, `k8s.pod.namespace`, `k8s.node.name`, `k8s.event.reason` | | 15 | `nccl_hang` | `pg_id`, `collective_seq_id`, `hanging_ranks` | | 16 | `xid_correlation` | `xid_code`, `node`, `evicted_pod` | diff --git a/module/sdk/verdict/decode_test.go b/module/sdk/verdict/decode_test.go index c363c682..1deb7a62 100644 --- a/module/sdk/verdict/decode_test.go +++ b/module/sdk/verdict/decode_test.go @@ -226,6 +226,19 @@ func TestDecode_AllShippedPatternsRoundTrip(t *testing.T) { "port": float64(1), "transition_count": float64(3), }}, + {"silent_data_corruption", map[string]any{ + "pattern.id": "13", "headline": "h", "remediation": "r", + "confidence": "full", + "kind": "vendor_signaled", + "evidence_trail": []any{ev("hw_gpu_sdc"), ev("gen_ai_training_eval_accuracy")}, + "gen_ai.training.job_id": "job-42", + "accuracy_drop": 0.15, + "baseline_accuracy": 0.95, + "observed_accuracy": 0.80, + "suspect_gpu_id": "0000:31:00.0", + "suspect_node": "n", + "sdc_counter_delta": float64(3), + }}, } for _, tc := range cases { diff --git a/module/sdk/verdict/schema.json b/module/sdk/verdict/schema.json index 1fbaa251..0711add3 100644 --- a/module/sdk/verdict/schema.json +++ b/module/sdk/verdict/schema.json @@ -49,7 +49,9 @@ "hw_fb", "cuda_oom", "pcie_aer", - "ib_port_state" + "ib_port_state", + "hw_gpu_sdc", + "gen_ai_training_eval_accuracy" ], "description": "Receiver-side surface the evidence came from. Per-pattern schemas tighten this to a pattern-specific subset (e.g. pod_evicted: [pod_event, node_condition]; pcie_aer: [pcie_aer, hw_io_collapse]). New kinds added here MUST land alongside a new EvidenceKind* constant in module/pkg/patterns/ and a per-pattern schema update." }, diff --git a/module/sdk/verdict/schema_sync_test.go b/module/sdk/verdict/schema_sync_test.go index a147438c..9adee0ee 100644 --- a/module/sdk/verdict/schema_sync_test.go +++ b/module/sdk/verdict/schema_sync_test.go @@ -3,6 +3,7 @@ package verdict_test import ( + "encoding/json" "os" "path/filepath" "testing" @@ -22,19 +23,30 @@ import ( // or, if the change is intentional, bump the SDK to a new schema // version (a new sibling package — never a silent SDK shape change). // +// Comparison runs on JSON-decoded structures (map[string]any) so +// whitespace-only reformatting of the canonical does NOT false-positive +// the gate; only semantic drift (added/removed/changed properties, +// enums, requireds) flips the test red. +// // This is the single-source-of-truth guard the criterion-12 brief // requires. func TestEmbeddedSchemaMatchesCanonical(t *testing.T) { t.Parallel() - embedded, err := os.ReadFile(filepath.Join(".", "schema.json")) //nolint:gosec // test-local + embeddedRaw, err := os.ReadFile(filepath.Join(".", "schema.json")) //nolint:gosec // test-local require.NoError(t, err, "module/sdk/verdict/schema.json must exist (build-time embed source)") - canonical, err := os.ReadFile( + canonicalRaw, err := os.ReadFile( filepath.Join("..", "..", "..", "docs", "schemas", "verdict-1.0.0-rc1.json")) //nolint:gosec // test-local require.NoError(t, err, "docs/schemas/verdict-1.0.0-rc1.json is the criterion-2 source of truth") - require.Equal(t, string(canonical), string(embedded), + var embedded, canonical any + require.NoError(t, json.Unmarshal(embeddedRaw, &embedded), + "module/sdk/verdict/schema.json must be valid JSON") + require.NoError(t, json.Unmarshal(canonicalRaw, &canonical), + "docs/schemas/verdict-1.0.0-rc1.json must be valid JSON") + + require.Equal(t, canonical, embedded, "module/sdk/verdict/schema.json drifted from docs/schemas/verdict-1.0.0-rc1.json. "+ "Run: cp docs/schemas/verdict-1.0.0-rc1.json module/sdk/verdict/schema.json") } diff --git a/module/sdk/verdict/verdict.go b/module/sdk/verdict/verdict.go index 8c2063f4..6be8eebf 100644 --- a/module/sdk/verdict/verdict.go +++ b/module/sdk/verdict/verdict.go @@ -41,7 +41,8 @@ type EvidenceRef struct { // Kind names the receiver-side surface the evidence came from. // v1.0-rc1 vocabulary: pod_event, node_condition, nccl_fr, // kernel_event, hw_error, hw_throttle, hw_io_collapse, hw_fb, - // cuda_oom, pcie_aer, ib_port_state. + // cuda_oom, pcie_aer, ib_port_state, hw_gpu_sdc, + // gen_ai_training_eval_accuracy. Kind string `json:"kind"` // UID is the upstream identifier — for Kubernetes Events it's the diff --git a/python/tracecore_verdict/schema.json b/python/tracecore_verdict/schema.json index 1fbaa251..0711add3 100644 --- a/python/tracecore_verdict/schema.json +++ b/python/tracecore_verdict/schema.json @@ -49,7 +49,9 @@ "hw_fb", "cuda_oom", "pcie_aer", - "ib_port_state" + "ib_port_state", + "hw_gpu_sdc", + "gen_ai_training_eval_accuracy" ], "description": "Receiver-side surface the evidence came from. Per-pattern schemas tighten this to a pattern-specific subset (e.g. pod_evicted: [pod_event, node_condition]; pcie_aer: [pcie_aer, hw_io_collapse]). New kinds added here MUST land alongside a new EvidenceKind* constant in module/pkg/patterns/ and a per-pattern schema update." }, diff --git a/python/tracecore_verdict/test_decode.py b/python/tracecore_verdict/test_decode.py index 053fa7c2..84d97e58 100644 --- a/python/tracecore_verdict/test_decode.py +++ b/python/tracecore_verdict/test_decode.py @@ -299,12 +299,27 @@ def _ev(kind: str) -> dict: "port": 1, "transition_count": 3, }, + "silent_data_corruption": { + "pattern.id": "13", + "headline": "h", + "remediation": "r", + "confidence": "full", + "kind": "vendor_signaled", + "evidence_trail": [_ev("hw_gpu_sdc"), _ev("gen_ai_training_eval_accuracy")], + "gen_ai.training.job_id": "job-42", + "accuracy_drop": 0.15, + "baseline_accuracy": 0.95, + "observed_accuracy": 0.80, + "suspect_gpu_id": "0000:31:00.0", + "suspect_node": "n", + "sdc_counter_delta": 3, + }, } @pytest.mark.parametrize("name,payload", list(SHIPPED_PATTERN_FIXTURES.items())) def test_decode_all_shipped_patterns_round_trip(name: str, payload: dict) -> None: - """All 8 shipped pattern verdicts decode cleanly.""" + """All 9 shipped pattern verdicts decode cleanly.""" v = decode(json.dumps(payload)) assert v.pattern_id == payload["pattern.id"] assert len(v.evidence_trail) >= 1 @@ -344,17 +359,22 @@ def test_evidence_ref_dataclass() -> None: def test_embedded_schema_matches_canonical() -> None: - """The package-data schema is byte-identical to docs/schemas/... + """The package-data schema is semantically identical to docs/schemas/... The Python build copies the canonical schema in alongside the package (single source of truth); drift fails the build at this test. + + Comparison runs on JSON-decoded dicts so whitespace-only + reformatting of the canonical does NOT false-positive the gate; + only semantic drift (added/removed/changed properties, enums, + requireds) flips the test red. """ pkg_dir = Path(__file__).parent - embedded = (pkg_dir / "schema.json").read_bytes() - canonical = ( - pkg_dir / ".." / ".." / "docs" / "schemas" / "verdict-1.0.0-rc1.json" - ).read_bytes() + embedded = json.loads((pkg_dir / "schema.json").read_text()) + canonical = json.loads( + (pkg_dir / ".." / ".." / "docs" / "schemas" / "verdict-1.0.0-rc1.json").read_text() + ) assert embedded == canonical, ( "python/tracecore_verdict/schema.json drifted from " "docs/schemas/verdict-1.0.0-rc1.json. Run: " From c54a35ea476b5fe2a711292290dc58c31730f4bb Mon Sep 17 00:00:00 2001 From: Tri Lam Date: Mon, 1 Jun 2026 13:09:15 -0700 Subject: [PATCH 9/9] fix(sdk): resync schema copies to canonical $id after main rebase Signed-off-by: Tri Lam --- module/sdk/verdict/schema.json | 2 +- python/tracecore_verdict/schema.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/module/sdk/verdict/schema.json b/module/sdk/verdict/schema.json index 0711add3..c68853b2 100644 --- a/module/sdk/verdict/schema.json +++ b/module/sdk/verdict/schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://schema.tracecore.io/verdict/1.0.0-rc1.json", + "$id": "https://raw.githubusercontent.com/TraceCoreAI/tracecore/main/docs/schemas/verdict-1.0.0-rc1.json", "title": "TraceCore Verdict envelope v1.0-rc1", "description": "Canonical envelope shape every TraceCore pattern detector emits. This is the v1.0-rc1 cut of the externally observable verdict contract — the shape downstream consumers (Prometheus alertmanager webhooks, Slack routers, audit pipelines) bind against. Per-pattern detectors layer pattern-specific fields on top of this envelope (e.g. `gpu_id`, `xid_code`, `kind`); per-pattern testdata schemas under module/pkg/patterns/testdata/_verdict.schema.json pin those extensions. The envelope intentionally sets `additionalProperties: true` so per-pattern extensions validate without re-listing here — drift in pattern-specific fields surfaces on the per-pattern schema, drift in envelope-level fields surfaces here. Versioning policy: docs/schemas/README.md; evolution policy: docs/DEPRECATION.md (forthcoming).", "type": "object", diff --git a/python/tracecore_verdict/schema.json b/python/tracecore_verdict/schema.json index 0711add3..c68853b2 100644 --- a/python/tracecore_verdict/schema.json +++ b/python/tracecore_verdict/schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://schema.tracecore.io/verdict/1.0.0-rc1.json", + "$id": "https://raw.githubusercontent.com/TraceCoreAI/tracecore/main/docs/schemas/verdict-1.0.0-rc1.json", "title": "TraceCore Verdict envelope v1.0-rc1", "description": "Canonical envelope shape every TraceCore pattern detector emits. This is the v1.0-rc1 cut of the externally observable verdict contract — the shape downstream consumers (Prometheus alertmanager webhooks, Slack routers, audit pipelines) bind against. Per-pattern detectors layer pattern-specific fields on top of this envelope (e.g. `gpu_id`, `xid_code`, `kind`); per-pattern testdata schemas under module/pkg/patterns/testdata/_verdict.schema.json pin those extensions. The envelope intentionally sets `additionalProperties: true` so per-pattern extensions validate without re-listing here — drift in pattern-specific fields surfaces on the per-pattern schema, drift in envelope-level fields surfaces here. Versioning policy: docs/schemas/README.md; evolution policy: docs/DEPRECATION.md (forthcoming).", "type": "object",