diff --git a/docs/rfcs/0012-kineto-receiver-scope.md b/docs/rfcs/0012-kineto-receiver-scope.md index cbaae4d2..997fce9e 100644 --- a/docs/rfcs/0012-kineto-receiver-scope.md +++ b/docs/rfcs/0012-kineto-receiver-scope.md @@ -236,7 +236,7 @@ Every error path calls `selftel.IncError(kind)` exactly once per file (not per e ### Resource bounds -Pinned in `pkg/kineto/decoder.go` package-level constants: +Pinned in `pkg/kineto/decoder.go` as package-level bounds: | Bound | Default | Derivation | |---|---|---| @@ -245,7 +245,7 @@ Pinned in `pkg/kineto/decoder.go` package-level constants: | `MaxStringBytes` | `1<<20` (1 MiB) | Per-field cap; matches the M11 precedent; defends against malformed `args.Stream` strings | | `MaxDepth` | `64` | Chrome-trace structure is flat; no nesting beyond `args.{}` | -Arithmetic for `MaxEvents`: `2.2e9 bytes / 140 bytes/event ≈ 15.7e6 events`, rounded up to the nearest power of two (`1<<24 = 16.78e6`). Exceed any bound triggers `ErrLimitExceeded`, aborts the file, bumps `IncError(KindLimitExceeded)`, and continues to the next file. +`MaxBytes` and `MaxEvents` are declared as `var` (not `const`) so tests can lower them to exercise the cap path on a tiny input. Production code does not mutate either; `MaxStringBytes` and `MaxDepth` remain `const`. Arithmetic for `MaxEvents`: `2.2e9 bytes / 140 bytes/event ≈ 15.7e6 events`, rounded up to the nearest power of two (`1<<24 = 16.78e6`). Exceed any bound triggers `ErrLimitExceeded`, aborts the file, bumps `IncError(KindLimitExceeded)`, and continues to the next file. ### `safe.Call` boundary diff --git a/pkg/kineto/decoder.go b/pkg/kineto/decoder.go index a4c9a932..a49a3003 100644 --- a/pkg/kineto/decoder.go +++ b/pkg/kineto/decoder.go @@ -16,9 +16,14 @@ import ( // See RFC-0012 §Resource bounds. var MaxBytes int64 = 1 << 32 // 4 GiB -// Per-event and per-field resource bounds. See RFC-0012 §Resource bounds. +// MaxEvents caps the number of traceEvents we will parse. Package-level +// var (not const); mirrors MaxBytes so tests can lower it to exercise +// the cap path without allocating a 16M-event input. Production code does +// not mutate it. See RFC-0012 §Resource bounds. +var MaxEvents = 1 << 24 // ~16.7M; derived from 2.2 GB / ~140 bytes-per-event median (pytorch#130622) + +// Per-field resource bounds. See RFC-0012 §Resource bounds. const ( - MaxEvents = 1 << 24 // ~16.7M; derived from 2.2 GB / ~140 bytes-per-event median (pytorch#130622) MaxStringBytes = 1 << 20 // 1 MiB per string field (Name/Cat/Ph/Args.ExternalID) MaxDepth = 64 // Chrome-trace structure is flat; 64 is generous ) diff --git a/pkg/kineto/decoder_test.go b/pkg/kineto/decoder_test.go index 5a154afc..3a5d6e24 100644 --- a/pkg/kineto/decoder_test.go +++ b/pkg/kineto/decoder_test.go @@ -81,12 +81,20 @@ func TestParse_VisitErrorPropagates(t *testing.T) { } func TestParse_MaxEventsCap(t *testing.T) { - if testing.Short() { - t.Skip("skipping MaxEvents stress under -short") - } + // Stub MaxEvents down to ~100 to exercise the cap path on a tiny + // input. Override pattern mirrors the MaxBytes test below + // (TestParse_MaxBytesCap_DistinguishedFromTruncated). The naive form + // (build MaxEvents+1 = 16M events as a ~1.1 GB JSON string and run + // the parser under -race) took ~400s and dominated verify-test + // wallclock; this preserves the assertion at sub-millisecond cost. + const stubCap = 100 + orig := MaxEvents + MaxEvents = stubCap + t.Cleanup(func() { MaxEvents = orig }) + var sb strings.Builder sb.WriteString(`{"traceEvents":[`) - for i := 0; i < MaxEvents+1; i++ { + for i := 0; i < stubCap+1; i++ { if i > 0 { sb.WriteString(",") }