Skip to content

fix(cli): reduce over-budget projected list pages instead of shortening values #175

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ysyneu merged 1 commit into from
Aug 29, 2026
Merged

fix(cli): reduce over-budget projected list pages instead of shortening values #175

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 14 additions & 15 deletions cmd/flashduty/main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,24 +112,23 @@ func TestSetVersionInfoBeforeExecute(t *testing.T) {
}
}

// Test 79: When a compact list projection overflows its byte budget, the
// binary exits non-zero, writes nothing to stdout, and reports the error on
// stderr — a pipeline reading stdout must see a failed call, never an empty
// page masquerading as "no data".
// Test 79: When a compact list projection cannot fit its byte budget at
// all — a single row that overflows on its own and carries nothing
// shortenable — the binary exits non-zero, writes nothing to stdout, and
// reports the error on stderr: a pipeline reading stdout must see a failed
// call, never an empty page masquerading as "no data". (A multi-row page
// that overflows is instead reduced to the leading rows that fit, announced
// on stderr.)
func TestProjectionOverflowFailsHard(t *testing.T) {
binPath := buildTestBinary(t, "")

// Stub the alert-event list endpoint with a page whose projection stays
// over the 16 KiB budget even after value shortening.
// Stub the alert-event list endpoint with one row whose projected labels
// blob alone exceeds the 16 KiB budget; a labels map is not a string, so
// no value shortening can rescue it.
var body strings.Builder
body.WriteString(`{"request_id":"r","error":{"code":"OK","message":""},"data":{"total":100,"items":[`)
for i := 0; i < 100; i++ {
if i > 0 {
body.WriteByte(',')
}
fmt.Fprintf(&body, `{"event_id":"%024x","alert_id":"%024x","event_severity":"Warning","event_status":"Triggered","event_time":1712000000,"title":%q}`,
i, i+1_000_000, strings.Repeat("x", 200))
}
body.WriteString(`{"request_id":"r","error":{"code":"OK","message":""},"data":{"total":1,"items":[`)
fmt.Fprintf(&body, `{"event_id":"%024x","alert_id":"%024x","event_severity":"Warning","event_status":"Triggered","event_time":1712000000,"title":"disk full","labels":{"payload":%q}}`,
1, 1_000_001, strings.Repeat("x", 20000))
body.WriteString(`]}}`)

srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
Expand All @@ -138,7 +137,7 @@ func TestProjectionOverflowFailsHard(t *testing.T) {
}))
defer srv.Close()

run := exec.Command(binPath, "alert-event", "list", "--limit", "100",
run := exec.Command(binPath, "alert-event", "list", "--fields", "event_id,labels",
"--output-format", "json", "--app-key", "test-key", "--base-url", srv.URL)
// Isolate HOME so the test never reads the developer's real CLI config.
run.Env = append(os.Environ(), "HOME="+t.TempDir())
Expand Down
13 changes: 9 additions & 4 deletions internal/cli/alert_event.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,12 +97,17 @@ func newAlertEventListCmd() *cobra.Command {
if err != nil {
return err
}
note, err := boundProjectedOutput(proj, compactListOutputLimit)
bounded, note, err := boundProjectedOutput(proj, compactListOutputLimit)
if err != nil {
return err
}
noteProjectionShortening(cmd.ErrOrStderr(), note)
return ctx.PrintList(proj, nil, len(result.Items), page, limit, int(result.Total))
proj = bounded.([]map[string]any)
noteProjectionBound(cmd.ErrOrStderr(), note)
effectiveLimit := limit
if len(proj) < len(result.Items) {
effectiveLimit = len(proj)
}
return ctx.PrintList(proj, nil, len(proj), page, effectiveLimit, int(result.Total))
}

return ctx.PrintList(result.Items, cols, len(result.Items), page, limit, int(result.Total))
Expand All @@ -119,7 +124,7 @@ func newAlertEventListCmd() *cobra.Command {
cmd.Flags().StringVar(&until, "until", "now", "End time")
cmd.Flags().IntVar(&limit, "limit", 20, "Max results (max 100)")
cmd.Flags().IntVar(&page, "page", 1, "Page number")
cmd.Flags().StringVar(&fields, "fields", "", "Comma-separated fields to project in json/toon output (e.g. event_id,alert_id,event_severity,event_status,event_time,title); ignored in table mode. Defaults to these compact event fields. Long strings are truncated as needed to keep structured output below 16 KiB.")
cmd.Flags().StringVar(&fields, "fields", "", "Comma-separated fields to project in json/toon output (e.g. event_id,alert_id,event_severity,event_status,event_time,title); ignored in table mode. Defaults to these compact event fields. If the page would exceed 16 KiB, only the leading rows that fit are emitted, with every value intact (announced on stderr).")

return cmd
}
5 changes: 3 additions & 2 deletions internal/cli/channel.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,11 +195,12 @@ func newChannelEscalateRuleListCmd() *cobra.Command {
if err != nil {
return err
}
note, err := boundProjectedOutput(proj, compactListOutputLimit)
bounded, note, err := boundProjectedOutput(proj, compactListOutputLimit)
if err != nil {
return err
}
noteProjectionShortening(cmd.ErrOrStderr(), note)
proj = bounded.([]map[string]any)
noteProjectionBound(cmd.ErrOrStderr(), note)
return ctx.PrintTotal(proj, nil, len(proj))
}

Expand Down
46 changes: 21 additions & 25 deletions internal/cli/command_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1705,24 +1705,22 @@ func TestCommandAlertListStructuredAnnouncesTruncation(t *testing.T) {
// ---------------------------------------------------------------------------

// TestCommandListProjectionOverflowFails pins that a compact list projection
// which cannot fit the byte budget fails the command instead of emitting
// anything: Execute returns the error and stdout stays empty, so a pipeline
// reading stdout sees a failed call, never an empty page masquerading as
// "no data".
// which cannot fit the byte budget at all — a single row that overflows on
// its own and carries nothing shortenable — fails the command instead of
// emitting anything: Execute returns the error and stdout stays empty, so a
// pipeline reading stdout sees a failed call, never an empty page
// masquerading as "no data". (A multi-row page that overflows is instead
// reduced to the leading rows that fit — see
// TestAlertEventListAutoReducesPageAtLargeLimit.)
func TestCommandListProjectionOverflowFails(t *testing.T) {
t.Run("incident list", func(t *testing.T) {
saveAndResetGlobals(t)
stub := newGFStub(t)
items := make([]any, 100)
for i := range items {
row := incidentRow()
row["incident_id"] = fmt.Sprintf("inc-%024d", i)
row["title"] = strings.Repeat("x", 200)
items[i] = row
}
stub.data = map[string]any{"items": items, "total": len(items)}
row := incidentRow()
row["labels"] = map[string]any{"payload": strings.Repeat("x", 20000)}
stub.data = map[string]any{"items": []any{row}, "total": 1}

out, stderrText, err := execCommandSplit("incident", "list", "--limit", "100", "--output-format", "json")
out, stderrText, err := execCommandSplit("incident", "list", "--fields", "incident_id,labels", "--output-format", "json")
if err == nil || !strings.Contains(err.Error(), "exceeds the 16384-byte limit") {
t.Fatalf("irreducible projection error = %v, want the byte-limit refusal", err)
}
Expand All @@ -1737,20 +1735,18 @@ func TestCommandListProjectionOverflowFails(t *testing.T) {
t.Run("alert-event list", func(t *testing.T) {
saveAndResetGlobals(t)
stub := newGFStub(t)
items := make([]any, 100)
for i := range items {
items[i] = map[string]any{
"event_id": fmt.Sprintf("%024x", i),
"alert_id": fmt.Sprintf("%024x", i+1_000_000),
"event_severity": "Warning",
"event_status": "Triggered",
"event_time": 1712000000 + i,
"title": strings.Repeat("x", 200),
}
row := map[string]any{
"event_id": fmt.Sprintf("%024x", 1),
"alert_id": fmt.Sprintf("%024x", 1_000_001),
"event_severity": "Warning",
"event_status": "Triggered",
"event_time": 1712000000,
"title": "disk full",
"labels": map[string]any{"payload": strings.Repeat("x", 20000)},
}
stub.data = map[string]any{"items": items, "total": len(items)}
stub.data = map[string]any{"items": []any{row}, "total": 1}

out, _, err := execCommandSplit("alert-event", "list", "--limit", "100", "--output-format", "json")
out, _, err := execCommandSplit("alert-event", "list", "--fields", "event_id,labels", "--output-format", "json")
if err == nil || !strings.Contains(err.Error(), "exceeds the 16384-byte limit") {
t.Fatalf("irreducible projection error = %v, want the byte-limit refusal", err)
}
Expand Down
120 changes: 85 additions & 35 deletions internal/cli/fieldproject.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,39 +80,42 @@ func noteDefaultProjection(w io.Writer, fields []string) {
strings.Join(fields, ","))
}

// noteProjectionShortening tells the caller, on stderr, that some values came
// back clipped. Without it a shortened value is only visible to a reader, not
// to the jq filter or exact match a --json consumer runs over it, so a query
// that silently matches nothing looks like an empty result rather than a
// truncated one.
func noteProjectionShortening(w io.Writer, note string) {
// noteProjectionBound relays a boundProjectedOutput note to the caller on
// stderr. Without it a reduced page or a shortened value is only visible to
// a reader, not to the jq filter or exact match a --json consumer runs over
// it, so a query that silently matches nothing looks like an empty result
// rather than a bounded one.
func noteProjectionBound(w io.Writer, note string) {
if note == "" {
return
}
_, _ = fmt.Fprintln(w, note)
}

// boundProjectedOutput keeps the new agent-oriented projections below their
// command budget without changing the selected keys. List rows (many small
// records) are shortened fairly when they overflow the budget, with
// shortened values marked with "...". A single-object detail projection is
// never modified: a truncated id or status string is indistinguishable from
// a genuinely short value, so silently shortening it would hand the caller
// wrong data instead of a compact one. If a detail projection doesn't fit,
// the command fails with an error instead.
// command budget without changing the selected keys. A list projection that
// overflows the budget is first reduced to the leading rows that fit with
// every value intact; only a single row that overflows the budget on its own
// is shortened fairly, with shortened values marked with "...". A
// single-object detail projection is never modified: a truncated id or
// status string is indistinguishable from a genuinely short value, so
// silently shortening it would hand the caller wrong data instead of a
// compact one. If a detail projection doesn't fit, the command fails with an
// error instead.
//
// It returns a caller-printable note (empty when nothing was shortened) that
// names the clipped fields, so the caller can announce the loss on stderr —
// the "..." marker is only visible to something that reads the value, never
// to the filter a --json consumer runs over it.
func boundProjectedOutput(data any, maxBytes int) (string, error) {
// It returns the bounded data with the same type it was given, plus a
// caller-printable note (empty when nothing was reduced or shortened), so
// the caller can announce the loss on stderr — the "..." marker is only
// visible to something that reads the value, never to the filter a --json
// consumer runs over it.
func boundProjectedOutput(data any, maxBytes int) (any, string, error) {
switch value := data.(type) {
case map[string]any:
return "", boundProjectedDetail(value, maxBytes)
return value, "", boundProjectedDetail(value, maxBytes)
case []map[string]any:
return boundProjectedList(value, maxBytes)
default:
return "", fmt.Errorf("internal error: unsupported projected output %T", data)
return nil, "", fmt.Errorf("internal error: unsupported projected output %T", data)
}
}

Expand Down Expand Up @@ -190,10 +193,14 @@ func isIdentifierField(key string) bool {
return strings.HasSuffix(key, "_id") || strings.HasSuffix(key, "_key")
}

// boundProjectedList shortens a list projection's string values fairly when
// the compact rows themselves overflow the budget: it finds the largest
// per-field byte cap that still makes everything fit, then applies that one
// cap to every shortenable string value across every row. A field already
// boundProjectedList keeps a list projection below the budget. A page that
// overflows is first reduced to the largest leading prefix of rows that
// fits, with every value intact: a --json consumer filters and matches on
// the values, so a partial page of intact rows serves it, while a full page
// of "..."-clipped rows silently defeats the filter. Only when one row alone
// overflows the budget does it shorten that row's string values fairly: it
// finds the largest per-field byte cap that still makes the row fit, then
// applies that one cap to every shortenable string value. A field already
// shorter than the cap is left completely untouched — only the field(s)
// actually responsible for the overflow (typically a long title) get
// shortened, each marked with "...". Identifier fields (keys ending in _id
Expand All @@ -202,27 +209,37 @@ func isIdentifierField(key string) bool {
// marker itself disappear, so a shortened value is always distinguishable
// from a genuinely short one; if no cap at or above that floor fits, the
// command fails with a small error instead of emitting values that look
// real but aren't. Whatever it clips, it reports back in the returned note.
func boundProjectedList(rows []map[string]any, maxBytes int) (string, error) {
// real but aren't. Whatever it reduces or clips, it reports back in the
// returned note.
func boundProjectedList(rows []map[string]any, maxBytes int) ([]map[string]any, string, error) {
encoded, err := marshalStructured(rows)
if err != nil {
return "", err
return nil, "", err
}
if len(encoded)+1 < maxBytes {
return "", nil
return rows, "", nil
}

// The overflow error names the fields responsible, exactly as the detail
// path does, so the request can be narrowed in one pass.
tooBig := func() (string, error) {
tooBig := func() ([]map[string]any, string, error) {
largest, err := largestProjectedFields(rows)
if err != nil {
return "", err
return nil, "", err
}
return "", fmt.Errorf("projected list is %d bytes across %d rows, exceeds the %d-byte limit; largest fields: %s; request fewer rows (--limit) or fewer --fields",
return nil, "", fmt.Errorf("projected list is %d bytes across %d rows, exceeds the %d-byte limit; largest fields: %s; request fewer rows (--limit) or fewer --fields",
len(encoded), len(rows), maxBytes, largest)
}

kept, err := largestFittingPrefix(rows, maxBytes)
if err != nil {
return nil, "", err
}
if kept > 0 {
return rows[:kept], fmt.Sprintf("note: emitted %d of %d projected rows (every value intact) to stay below the %d-byte structured-output limit; narrow --fields or lower --limit to fit more rows per page — the rows past the first %d were not emitted",
kept, len(rows), maxBytes, kept), nil
}

maxLen := 0
for _, row := range rows {
for key, value := range row {
Expand Down Expand Up @@ -268,7 +285,7 @@ func boundProjectedList(rows []map[string]any, maxBytes int) (string, error) {
return tooBig()
}
if ok, err := fits(minMarkedTruncationCap); err != nil {
return "", err
return nil, "", err
} else if !ok {
return tooBig()
}
Expand All @@ -282,7 +299,7 @@ func boundProjectedList(rows []map[string]any, maxBytes int) (string, error) {
mid := lo + (hi-lo+1)/2
ok, err := fits(mid)
if err != nil {
return "", err
return nil, "", err
}
if ok {
lo = mid
Expand Down Expand Up @@ -312,17 +329,50 @@ func boundProjectedList(rows []map[string]any, maxBytes int) (string, error) {
}
}
if shortened == 0 {
return "", nil
return rows, "", nil
}
names := make([]string, 0, len(fields))
for name := range fields {
names = append(names, name)
}
sort.Strings(names)
return fmt.Sprintf("note: %d of %d string values were shortened to fit the %d-byte limit and now end with \"...\" (fields: %s); matching or filtering on those fields will miss — narrow --fields or --limit for untruncated values",
return rows, fmt.Sprintf("note: %d of %d string values were shortened to fit the %d-byte limit and now end with \"...\" (fields: %s); matching or filtering on those fields will miss — narrow --fields or --limit for untruncated values",
shortened, total, maxBytes, strings.Join(names, ", ")), nil
}

// largestFittingPrefix returns the largest n < len(rows) whose encoded prefix
// rows[:n] fits the budget, or 0 when even one row overflows it. Prefix size
// is monotone — appending a row never shrinks the encoding — so the boundary
// is found by binary search between lo=1 (known fitting, checked first) and
// hi=len(rows) (known not to fit: the caller only reaches here on overflow).
func largestFittingPrefix(rows []map[string]any, maxBytes int) (int, error) {
fits := func(n int) (bool, error) {
encoded, err := marshalStructured(rows[:n])
if err != nil {
return false, err
}
return len(encoded)+1 < maxBytes, nil
}
ok, err := fits(1)
if err != nil || !ok {
return 0, err
}
lo, hi := 1, len(rows) // fits(lo) holds, fits(hi) does not
for hi-lo > 1 {
mid := lo + (hi-lo)/2
ok, err := fits(mid)
if err != nil {
return 0, err
}
if ok {
lo = mid
} else {
hi = mid
}
}
return lo, nil
}

func truncateUTF8Bytes(value string, maxBytes int) string {
if len(value) <= maxBytes {
return value
Expand Down
Loading