From f237c5a2667d1f49fd94d417d9b1581f6272a5c1 Mon Sep 17 00:00:00 2001 From: Masen Furer Date: Thu, 23 Jul 2026 22:26:09 +0000 Subject: [PATCH 1/5] =?UTF-8?q?Resolve=20bin=20colors=20once=20per=20trace?= =?UTF-8?q?,=20never=20per=20density=20request=20(LOD=20doc=20=C2=A72)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit density_view resolved the kernel's mean-color source — the full-column LUT-index/RGBA quantize — at the top of every request, before the tier decision, even though pyramid replies compose prebuilt color planes and point drills ship sliced channels; only the two bin_2d_mean_color branches ever consume it. On the 100M-point FastAPI drilldown demo that O(N) NumPy pass (multi-GB temporaries) cost 1.3-7 s per reply, 10-100x the actual tier work (pyramid compose 3-34 ms, drill scans 100-230 ms, mid-band re-bin ~360 ms on a 4-core host). The resolution is a pure function of the immutable channel values and their global domain, so cache it on the trace (interaction.trace_bin_colors; None/0/dict, the _pyr_handle idiom) and share it across every consumer: pyramid build, first-paint emit, exact and no-rescan re-bins. density_view now decides only the cheap bins_mean_color fact up front and materializes the source solely in the branches that feed bin_2d_mean_color. Appends drop the cache beside the other pre-append derived state and the append's own refresh emit re-resolves it, once, over the post-append column. The cache is a rebuildable derived buffer (dossier §27), itemized as memory_report()['bin_color_bytes'] and counted in resident_array_bytes. Measured on the demo: every request now completes in 0.02-0.45 s with byte-identical replies (pyramid band 0-38 ms), and figure warmup no longer pays the resolve twice. --- CHANGELOG.md | 12 ++++ python/xy/_figure.py | 6 +- python/xy/_payload.py | 6 +- python/xy/_trace.py | 9 +++ python/xy/interaction.py | 54 +++++++++++++-- spec/design-dossier.md | 2 +- spec/design/lod-architecture.md | 16 ++++- tests/test_density_mean_color.py | 110 +++++++++++++++++++++++++++++++ 8 files changed, 206 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b136839..d6326753 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,18 @@ in the README). to the internal engine object. ### Changed +- **Bin-color resolution is resolved once per trace, never per request + (LOD doc §2).** `density_view` used to quantize the *entire* color column + into the kernel's LUT-index/RGBA source on every reply — an O(N) NumPy pass + with multi-GB temporaries that cost the 100M-point FastAPI drilldown demo + 1.3–7 s per request, ~10–100× the actual tier work, even for pyramid and + points-band replies that never consume it. The resolution is now cached on + the trace (`interaction.trace_bin_colors`; a rebuildable §27 derived buffer, + itemized as `memory_report()["bin_color_bytes"]`, invalidated by appends) + and materialized only by the branches that feed `bin_2d_mean_color`. On the + demo host every drilldown request now completes in 0.02–0.45 s with + byte-identical replies; `test_adaptive_drilldown_cycle_mean_color` guards + the contract in CodSpeed. - **No sampled points above the resolution of the graph (#225).** Interactive `density_view` replies no longer ship a point-sample overlay: a fixed-size sample above the drill budget reads as individual data points at a zoom diff --git a/python/xy/_figure.py b/python/xy/_figure.py index dcad0766..7689f32d 100644 --- a/python/xy/_figure.py +++ b/python/xy/_figure.py @@ -1830,8 +1830,12 @@ def memory_report(self) -> dict[str, Any]: n_total = sum(t.n_points for t in self.traces) or 1 report["transport_bytes_per_point"] = len(blob) / n_total report["pyramid_bytes"] = interaction.pyramid_report_bytes(self) + report["bin_color_bytes"] = interaction.bin_color_cache_bytes(self) report["resident_array_bytes"] = ( - report["canonical_bytes"] + report["channel_bytes"] + report["pyramid_bytes"] + report["canonical_bytes"] + + report["channel_bytes"] + + report["pyramid_bytes"] + + report["bin_color_bytes"] ) report["backend"] = kernels.BACKEND return report diff --git a/python/xy/_payload.py b/python/xy/_payload.py index a992b0ef..19c149ff 100644 --- a/python/xy/_payload.py +++ b/python/xy/_payload.py @@ -10,7 +10,7 @@ import numpy as np -from . import _native, channels, kernels, lod +from . import _native, channels, interaction, kernels, lod from ._trace import Trace from .columns import Column from .config import ( @@ -1062,7 +1062,9 @@ def _density_trace_spec(self, t: Trace, xr, yr, w, h, pw: "_PayloadWriter") -> d else channels.DEFAULT_COLORMAP ) dropped_channels = list(t.per_item_channel_names()) - bin_colors = channels.resolve_bin_colors(t.color_ch, None, DEFAULT_PALETTE) + # Cached full-column resolution (LOD doc §2): the O(N) quantize pass + # is shared with the pyramid build and every later grid reply. + bin_colors = interaction.trace_bin_colors(t) density = { "buf": pw.ship_u8(encoded_grid), "w": w, diff --git a/python/xy/_trace.py b/python/xy/_trace.py index b3b82c33..46f96c36 100644 --- a/python/xy/_trace.py +++ b/python/xy/_trace.py @@ -79,6 +79,15 @@ class Trace: drill_history: dict[int, Any] = field( default_factory=dict, init=False, repr=False, compare=False ) + # Full-column kernel color source for mean-color density binning (LOD doc + # §2), managed by `interaction.trace_bin_colors`: None = never resolved, + # 0 = resolved and not applicable (no channel / constant), otherwise the + # `resolve_bin_colors` kwargs dict. Resolving quantizes every canonical row + # — an O(N) pass — while the result is a pure function of the immutable + # channel values and their global domain, so it is computed once and shared + # by every grid consumer (pyramid build, exact re-bins, first-paint emit). + # A rebuildable derived cache (§27); appends invalidate it. + _bin_colors: Any = field(default=None, init=False, repr=False, compare=False) # Count-pyramid cache (§5 Tier 3), managed by `interaction.py`: None = # never tried, 0 = tried and not applicable, otherwise the native handle. # The finalizer frees the native side when the trace is collected. diff --git a/python/xy/interaction.py b/python/xy/interaction.py index feb3b140..d19b903d 100644 --- a/python/xy/interaction.py +++ b/python/xy/interaction.py @@ -385,6 +385,40 @@ def _pyramid_base_dim_for(t: Trace) -> int: return int(min(PYRAMID_MAX_DIM, max(PYRAMID_BASE_DIM, pow2))) +def trace_bin_colors(t: Trace) -> Optional[dict]: + """The trace's full-column kernel color source for mean-color binning + (LOD doc §2), resolved once and cached on the trace. + + `channels.resolve_bin_colors` over the full column quantizes every + canonical row — an O(N) pass whose NumPy temporaries reach multiple GB on + a 100M-point trace — while its result depends only on the channel's + immutable values and global domain. Cache it so each of its consumers + (pyramid build, the exact and no-rescan mean-color re-bins, the + first-paint density emit) pays that pass once per trace instead of once + per request; `density_view` must never resolve it for a reply that ships + no mean-color grid (pyramid replies use the prebuilt color planes, drills + ship sliced channels). A rebuildable derived cache (§27), dropped by + `append_data` (appended rows' colors and a possibly moved channel domain + both change the resolution).""" + cached = t._bin_colors + if cached is None: + resolved = channels.resolve_bin_colors(t.color_ch, None, DEFAULT_PALETTE) + t._bin_colors = 0 if resolved is None else resolved + return resolved + return None if cached == 0 else cached + + +def bin_color_cache_bytes(fig: Any) -> int: + """Memory-report line (design dossier §27): bytes held by resolved + bin-color caches (per-row idx/rgba planes plus their LUTs).""" + total = 0 + for t in fig.traces: + cached = getattr(t, "_bin_colors", None) + if cached: + total += sum(int(v.nbytes) for v in cached.values() if isinstance(v, np.ndarray)) + return total + + def _ensure_pyramid(t: Trace) -> int | None: """Lazily build the trace's count pyramid (§5 Tier 3). Cached on the trace; 0 is remembered as "tried and not applicable" so we never rebuild. @@ -412,7 +446,7 @@ def _ensure_pyramid(t: Trace) -> int | None: x1 += (x1 - x0) * 1e-9 y1 += (y1 - y0) * 1e-9 base_dim = _pyramid_base_dim_for(t) - bin_colors = channels.resolve_bin_colors(t.color_ch, None, DEFAULT_PALETTE) + bin_colors = trace_bin_colors(t) if bin_colors is not None: handle = kernels.pyramid_build_color( t.x.values, t.y.values, x0, x1, y0, y1, base_dim, **bin_colors @@ -688,9 +722,16 @@ def density_view( binning = "exact" grid = None # Mean point color plane (LOD doc §2): channel-bearing traces ship it - # alongside the counts wherever a grid is produced below. + # alongside the counts wherever a grid is produced below. Only the cheap + # does-it-bin-colors fact is decided up front; the full-column idx/lut + # source (`trace_bin_colors` — an O(N) resolve, cached on the trace) is + # materialized solely by the branches that feed `bin_2d_mean_color`. + # Pyramid replies compose prebuilt color planes and point drills ship + # sliced channels, so resolving here would charge every request a + # full-column pass those tiers never consume (the 100M drilldown demo + # paid 1–2 s per reply for it). rgba_grid: np.ndarray | None = None - bin_colors = channels.resolve_bin_colors(t.color_ch, None, DEFAULT_PALETTE) + mean_colors = channels.bins_mean_color(t.color_ch) # Texture sampling for the density grid: "nearest" when the grid is at full # screen resolution (exact deep-zoom detail — crisp, no interpolation bleed), # "linear" when it is upsampled from a coarser tier (smooth aggregate). @@ -763,7 +804,7 @@ def density_view( # are affordable, gather **once** and decide by the *actual* in-window count. if ( sidx is not None - and bin_colors is None + and not mean_colors and (grid is None or binning.endswith("-upsampled")) and sidx.window_count(lo_x, hi_x, lo_y, hi_y) <= SPATIAL_EXACT_MAX_POINTS ): @@ -806,6 +847,7 @@ def density_view( visible = plan.visible w, h = plan.grid_w, plan.grid_h grid = kernels.bin_2d(xv, yv, lo_x, hi_x, lo_y, hi_y, w, h) + bin_colors = trace_bin_colors(t) if mean_colors else None if bin_colors is not None: # This branch is already the O(N) correctness net; the mean-color # pass (LOD doc §2) rides the same full-column scan cost. @@ -837,6 +879,7 @@ def density_view( bx, (bx0, bx1) = fig._binning_coords(t.x_axis, xv, (lo_x, hi_x)) by, (by0, by1) = fig._binning_coords(t.y_axis, yv, (lo_y, hi_y)) grid = kernels.bin_2d(bx, by, bx0, bx1, by0, by1, w, h) + bin_colors = trace_bin_colors(t) if mean_colors else None if bin_colors is not None: # Mean point color per cell (LOD doc §2): same window, same # binning space, occupied cells match the count grid exactly. @@ -1186,6 +1229,9 @@ def _style_tail(name: str, values: Any) -> Optional[np.ndarray]: # The trace crossed the lazy-index threshold after an earlier # "not applicable" result; let the next wide view build it. t._pyr_handle = None + # The cached bin-color resolution covered the pre-append rows over the + # pre-append domain; drop it for a lazy full re-resolve (LOD doc §2). + t._bin_colors = None lod.exit_drill(t) # Remembered subsets were computed against the pre-append canonical state; # the client rebuilds its GPU traces (and drops its cached point windows) diff --git a/spec/design-dossier.md b/spec/design-dossier.md index 07c0068e..b6fdc514 100644 --- a/spec/design-dossier.md +++ b/spec/design-dossier.md @@ -951,7 +951,7 @@ The five classes of memory, per chart: | Class | Lives in | Sized by | Freed when | |---|---|---|---| | **Canonical columns** | JS ArrayBuffers / mmap (native) / server (Tier 3) — *never* WASM linear memory | data | trace removed (or explicitly demoted, below) | -| **Derived buffers** (decimations, pyramid tiles, segment indices) | worker-side buffers + LodCache | screen (per entry) × cache budget | LRU-evicted under byte budget; always recomputable | +| **Derived buffers** (decimations, pyramid tiles, bin-color resolutions, segment indices) | worker-side buffers + LodCache | screen (per entry) × cache budget | LRU-evicted under byte budget; always recomputable | | **Staging** (encode/upload scratch) | WASM arena + mapped GPU staging rings | screen, fixed | reused every frame — never grows with data | | **GPU buffers/textures** | VRAM | visible working set | evicted under VRAM budget; rebuilt from canonical + derived on demand or device-loss | | **Overheads** | everywhere | — | *counted, not ignored*: validity bitmaps (1 bit/val), dictionaries, per-buffer GPU alignment padding (256 B granularity), double-buffering during in-flight uploads | diff --git a/spec/design/lod-architecture.md b/spec/design/lod-architecture.md index a83082cf..80592a43 100644 --- a/spec/design/lod-architecture.md +++ b/spec/design/lod-architecture.md @@ -140,6 +140,17 @@ lightness (the points' own alpha compositing). re-bin (the count grid and sample selection stay fused as before; plain scatters are byte-identical to the pre-color pipeline), +4 B/cell on the wire, and the §4 pyramid's color planes at +8 B/cell. + The kernel's per-point color *source* (LUT indices or packed RGBA8) is a + full-column O(N) quantize over immutable channel values and their global + domain, so it is resolved **once per trace** and cached + (`interaction.trace_bin_colors`; a rebuildable §27 derived buffer, + itemized as `memory_report()["bin_color_bytes"]`, dropped on append and + lazily re-resolved by the append's own refresh emit). `density_view` + resolves it only in the branches that feed `bin_2d_mean_color` — pyramid + replies compose the §4 prebuilt color planes and point drills ship sliced + channels, so a per-request or per-tier resolve is a regression (it charged + the 100M FastAPI drilldown demo 1–2 s per reply, ~10–100× the tier work; + tripwire: `test_adaptive_drilldown_cycle_mean_color`). *Why mean-of-colors and not colormap(mean value):* the mean color is representation-agnostic (categories have no mean value), matches the drilled points' downsample exactly (the anti-jarring contract, T3), and @@ -603,7 +614,10 @@ contract entry before it lands. LUT indices + a ≤256-entry RGBA8 LUT (continuous quantizes to the client's 256 texels; categorical passes codes, wide codes fold modulo the palette) or per-point straight RGBA8 (`direct_rgba`) — - `channels.resolve_bin_colors` maps every channel mode onto one of the two. + `channels.resolve_bin_colors` maps every channel mode onto one of the two, + and `interaction.trace_bin_colors` caches the full-column resolution per + trace (§2 cost note) so pyramid build, first-paint emit, and every exact + re-bin share one O(N) quantize instead of paying it per request. 2. **Done:** mean-color planes wired through the initial emit (`_payload._density_trace_spec`), `density_view` (exact and pyramid paths), the static SVG/PNG exporters, and the standalone re-bin worker; diff --git a/tests/test_density_mean_color.py b/tests/test_density_mean_color.py index 84ec2add..db014d9c 100644 --- a/tests/test_density_mean_color.py +++ b/tests/test_density_mean_color.py @@ -346,3 +346,113 @@ def test_wide_categorical_codes_fold_onto_palette(): assert out["lut"].shape[0] == len(DEFAULT_PALETTE) expected = (codes % len(DEFAULT_PALETTE)).astype(np.uint8) assert np.array_equal(out["idx"], expected) + + +# --- resolution cost: once per trace, never per request ---------------------- +# The full-column resolve quantizes every canonical row (O(N) with large NumPy +# temporaries — 1-2 s/request on the 100M FastAPI drilldown demo before it was +# cached). These tests pin the contract: at most one resolve per trace, shared +# by every consumer, refreshed exactly once by an append, and never triggered +# by a reply that ships no mean-color grid. + + +def _count_resolves(monkeypatch) -> dict[str, int]: + calls = {"n": 0} + original = channels.resolve_bin_colors + + def counting(cc, sel, palette): + calls["n"] += 1 + return original(cc, sel, palette) + + monkeypatch.setattr(channels, "resolve_bin_colors", counting) + return calls + + +def test_density_view_exact_band_resolves_colors_once(monkeypatch): + calls = _count_resolves(monkeypatch) + n = SCATTER_DENSITY_THRESHOLD * 3 + rng = np.random.default_rng(11) + x = rng.uniform(0.0, 100.0, n) + y = rng.uniform(0.0, 100.0, n) + fig = Figure().scatter(x, y, color=x.copy(), density=True) + first, first_bufs = fig.density_view(0, 0.0, 100.0, 0.0, 100.0, 256, 192) + assert calls["n"] == 1 + again, again_bufs = fig.density_view(0, 0.0, 90.0, 0.0, 90.0, 256, 192) + assert calls["n"] == 1 # served from the trace cache, not re-resolved + for update, bufs in ((first, first_bufs), (again, again_bufs)): + d = update["traces"][0]["density"] + assert d["color_agg"] == "mean" and len(bufs[d["rgba"]]) # plane still ships + + +def test_density_view_points_band_never_resolves_colors(monkeypatch): + calls = _count_resolves(monkeypatch) + n = SCATTER_DENSITY_THRESHOLD * 3 + rng = np.random.default_rng(12) + x = rng.uniform(0.0, 100.0, n) + y = rng.uniform(0.0, 100.0, n) + fig = Figure().scatter(x, y, color=x.copy(), density=True) + upd, _ = fig.density_view(0, 0.0, 20.0, 0.0, 20.0, 256, 192) + tr = upd["traces"][0] + assert tr["mode"] == "points" + assert tr["color"]["mode"] == "continuous" # channels restored on drill + assert calls["n"] == 0 # drills ship sliced channels; no full-column pass + + +def test_pyramid_band_reuses_build_time_resolution(monkeypatch): + calls = _count_resolves(monkeypatch) + n = PYRAMID_MIN_POINTS + 50_000 + rng = np.random.default_rng(13) + x = rng.uniform(0.0, 10.0, n) + y = rng.uniform(0.0, 1.0, n) + fig = Figure().scatter(x, y, color=x.copy()) + upd, _ = fig.density_view(0, 0.0, 10.0, 0.0, 1.0, 128, 96) + tr = upd["traces"][0] + assert tr["binning"].startswith("pyramid-L") + assert tr["density"]["color_agg"] == "mean" + assert calls["n"] == 1 # the colored pyramid build resolved (and cached) it + fig.density_view(0, 0.5, 9.5, 0.0, 1.0, 128, 96) + assert calls["n"] == 1 # composes prebuilt color planes; no re-resolve + + +def test_append_re_resolves_bin_colors_exactly_once(monkeypatch): + calls = _count_resolves(monkeypatch) + n = SCATTER_DENSITY_THRESHOLD * 3 + rng = np.random.default_rng(14) + x = rng.uniform(0.0, 100.0, n) + y = rng.uniform(0.0, 100.0, n) + fig = Figure().scatter(x, y, color=x.copy(), density=True) + fig.density_view(0, 0.0, 100.0, 0.0, 100.0, 256, 192) + assert calls["n"] == 1 + t = fig.traces[0] + assert len(t._bin_colors["idx"]) == n + # Appending invalidates the cache; the refresh payload the append emits + # re-resolves over the post-append column — once — and later grid replies + # reuse that fresh resolution. + fig.append(0, [50.0], [50.0], color=[123.0]) + assert calls["n"] == 2 + assert len(t._bin_colors["idx"]) == n + 1 + fig.density_view(0, 0.0, 100.0, 0.0, 100.0, 256, 192) + assert calls["n"] == 2 + + +def test_memory_report_itemizes_bin_color_cache_bytes(): + n = SCATTER_DENSITY_THRESHOLD * 3 + rng = np.random.default_rng(15) + fig = Figure().scatter( + rng.uniform(0.0, 100.0, n), + rng.uniform(0.0, 100.0, n), + color=np.arange(n, dtype=float), + density=True, + ) + report = fig.memory_report() # build_payload inside resolves + caches + # Continuous channel: n u8 LUT indices + the (256, 4) RGBA8 LUT. + assert report["bin_color_bytes"] == n + 256 * 4 + assert ( + report["resident_array_bytes"] + == report["canonical_bytes"] + + report["channel_bytes"] + + report["pyramid_bytes"] + + report["bin_color_bytes"] + ) + plain = Figure().scatter(np.arange(1000.0), np.arange(1000.0)) + assert plain.memory_report()["bin_color_bytes"] == 0 From 1fcbca95f75f66630955b35e0e7c60fc39bd1a29 Mon Sep 17 00:00:00 2001 From: Masen Furer Date: Thu, 23 Jul 2026 22:26:20 +0000 Subject: [PATCH 2/5] Benchmark the colored drilldown cycle; expect the T13 window in the plain one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_adaptive_drilldown_cycle asserted the deep reply's visible count against the raw 0..10 view window, but a drill ships the widest aligned window that still fits the budget (T13), so visible counts the padded window and the row has been failing since padded drills landed (CodSpeed runs on pull requests only, and this branch has none yet). Derive the expectation from a warm reply's own x_range/y_range instead — exact without hardcoding the pad ladder. Add test_adaptive_drilldown_cycle_mean_color, the channel-bearing twin the suite was missing: the drilldown cycle on a continuous-color trace (the FastAPI demo shape) with a mid-band exact re-bin leg, asserting the mean-color plane rides the pyramid and exact replies and that the drill restores per-point channels. Every reply must reuse the trace-cached bin-color resolution; re-resolving the column per request multiplies the row by the column length (2.25x wall clock at the benchmark's 2.1M rows, 1.3-7 s per reply on the 100M demo), which is exactly the regression the previous commit removed — with no colored row, CodSpeed could not see it. Row counts in the methodology doc move 73->74 and 102->103; the category registry and results tracking list name the new row. --- benchmarks/categories.py | 2 +- benchmarks/test_codspeed_kernels.py | 112 ++++++++++++++++++++++++---- spec/benchmarks/methodology.md | 14 +++- spec/benchmarks/results.md | 12 ++- 4 files changed, 118 insertions(+), 22 deletions(-) diff --git a/benchmarks/categories.py b/benchmarks/categories.py index ff6b0aa6..6db43d9f 100644 --- a/benchmarks/categories.py +++ b/benchmarks/categories.py @@ -50,7 +50,7 @@ "name": "Adaptive scatter drilldown", "why": "The large-data claim needs a credible path from overview to exact visible points.", "metrics": "visible-query latency, tier-switch latency, exact-point recovery, badge accuracy", - "harness": "benchmarks/test_codspeed_kernels.py::test_adaptive_drilldown_cycle", + "harness": "benchmarks/test_codspeed_kernels.py::test_adaptive_drilldown_cycle and ::test_adaptive_drilldown_cycle_mean_color (channel-bearing: per-request cost must exclude full-column color work)", "status": "tracked", "goal": "Exact points when visible count is under budget; sampled/density with explicit counts otherwise.", }, diff --git a/benchmarks/test_codspeed_kernels.py b/benchmarks/test_codspeed_kernels.py index 5eee704a..43a6b7f8 100644 --- a/benchmarks/test_codspeed_kernels.py +++ b/benchmarks/test_codspeed_kernels.py @@ -225,6 +225,29 @@ def compatibility_kernel_data() -> dict[str, np.ndarray]: } +def _expect_drill_window(fig: Figure, x: np.ndarray, y: np.ndarray) -> None: + """Stash the deep view's expected drill window and count on the figure. + + A drill reply ships the widest ALIGNED window around the view that still + fits the budget (LOD doc T13), so `visible` counts the SHIPPED window — + the reply's own `x_range`/`y_range` — not the raw view. Alignment is a + pure function of the extent and the view's span bucket, so every + benchmark iteration resolves to the same window and the same exact count; + reading both from a warm reply keeps the assertions exact without + hardcoding the pad ladder. + """ + deep, _ = fig.density_view(0, 0.0, 10.0, 0.0, 10.0, GRID_W, GRID_H) + trace = deep["traces"][0] + assert trace["mode"] == "points" + (wx0, wx1), (wy0, wy1) = trace["x_range"], trace["y_range"] + assert wx0 <= 0.0 and wx1 >= 10.0 and wy0 <= 0.0 and wy1 >= 10.0 + fig._benchmark_deep_window = (wx0, wx1, wy0, wy1) + fig._benchmark_deep_expected = int( + np.count_nonzero((x >= wx0) & (x <= wx1) & (y >= wy0) & (y <= wy1)) + ) + assert fig._benchmark_deep_expected == trace["visible"] + + @pytest.fixture(scope="module") def drilldown_figure() -> Figure: rng = np.random.default_rng(17) @@ -235,22 +258,33 @@ def drilldown_figure() -> Figure: # Warm the lazily-built pyramid so CodSpeed tracks interactive viewport # refresh cost, not one-time index construction. fig.density_view(0, 0.0, 100.0, 0.0, 100.0, GRID_W, GRID_H) - deep, _ = fig.density_view(0, 0.0, 10.0, 0.0, 10.0, GRID_W, GRID_H) + _expect_drill_window(fig, x, y) fig.density_view(0, 0.0, 100.0, 0.0, 100.0, GRID_W, GRID_H) + return fig - # The drill ships the widest ALIGNED window around the view that still - # fits the budget (LOD doc T13), so the count oracle is the SHIPPED - # window's, taken from the warm-up reply — alignment is deterministic, - # every benchmark iteration resolves to the same window. - trace = deep["traces"][0] - assert trace["mode"] == "points" - (wx0, wx1), (wy0, wy1) = trace["x_range"], trace["y_range"] - assert wx0 <= 0.0 and wx1 >= 10.0 and wy0 <= 0.0 and wy1 >= 10.0 - fig._benchmark_deep_window = (wx0, wx1, wy0, wy1) - fig._benchmark_deep_expected = int( - np.count_nonzero((x >= wx0) & (x <= wx1) & (y >= wy0) & (y <= wy1)) - ) - assert fig._benchmark_deep_expected == trace["visible"] + +@pytest.fixture(scope="module") +def colored_drilldown_figure() -> Figure: + """The drilldown workload with a continuous color channel — the FastAPI + 100M-demo shape, where every grid reply carries the mean-color plane + (LOD doc §2) and drills restore per-point channels. + + Warm calls build the colored pyramid AND the trace's cached full-column + color resolution (`interaction.trace_bin_colors`), so the measured region + tracks steady-state per-request cost. That cost must stay free of any + O(N) full-column channel pass: before the resolution was cached, every + reply of every tier re-quantized the whole color column, costing the 100M + drilldown demo 1-2 s per request against ~10-400 ms of real tier work. + """ + rng = np.random.default_rng(23) + x = rng.uniform(0.0, 100.0, DRILL_N).astype(np.float64, copy=False) + y = rng.uniform(0.0, 100.0, DRILL_N).astype(np.float64, copy=False) + color = np.hypot(x - 50.0, y - 50.0) + fig = xy.chart(xy.scatter(x=x, y=y, color=color, colormap="viridis", density=True)).figure() + + fig.density_view(0, 0.0, 100.0, 0.0, 100.0, GRID_W, GRID_H) + _expect_drill_window(fig, x, y) + fig.density_view(0, 0.0, 100.0, 0.0, 100.0, GRID_W, GRID_H) return fig @@ -1112,6 +1146,56 @@ def test_adaptive_drilldown_cycle(benchmark, drilldown_figure): assert result > DRILL_N +def _colored_drilldown_cycle(fig: Figure) -> int: + wide, wide_buffers = fig.density_view(0, 0.0, 100.0, 0.0, 100.0, GRID_W, GRID_H) + # ~12.5% of the uniform domain: between the drill budget and the pyramid + # margin, so this window takes the exact re-bin plus its mean-color scan. + mid, mid_buffers = fig.density_view(0, 0.0, 39.0, 0.0, 32.0, GRID_W, GRID_H) + deep, deep_buffers = fig.density_view(0, 0.0, 10.0, 0.0, 10.0, GRID_W, GRID_H) + + wide_trace = wide["traces"][0] + mid_trace = mid["traces"][0] + deep_trace = deep["traces"][0] + assert wide_trace["mode"] == "density" + assert str(wide_trace.get("binning", "")).startswith("pyramid-L") + assert wide_trace["density"].get("color_agg") == "mean" # prebuilt planes rode along + assert mid_trace["mode"] == "density" + assert mid_trace["binning"] == "exact" + assert mid_trace["density"].get("color_agg") == "mean" # cached-idx mean-color scan + assert deep_trace["mode"] == "points" + # T13: the shipped window is the deterministic padded aligned superset of + # the view, so both the window and its exact count repeat every cycle. + assert tuple(deep_trace["x_range"]) == fig._benchmark_deep_window[:2] + assert tuple(deep_trace["y_range"]) == fig._benchmark_deep_window[2:] + assert deep_trace["visible"] == fig._benchmark_deep_expected + assert deep_trace["color"]["mode"] == "continuous" # drill restores channels + + back, back_buffers = fig.density_view(0, 0.0, 100.0, 0.0, 100.0, GRID_W, GRID_H) + back_trace = back["traces"][0] + assert back_trace["mode"] == "density" + assert str(back_trace.get("binning", "")).startswith("pyramid-L") + return ( + int(wide_trace["visible"]) + + int(mid_trace["visible"]) + + int(deep_trace["visible"]) + + int(back_trace["visible"]) + + sum(len(buffer) for buffer in wide_buffers + mid_buffers + deep_buffers + back_buffers) + ) + + +def test_adaptive_drilldown_cycle_mean_color(benchmark, colored_drilldown_figure): + """The drilldown cycle on a channel-bearing trace (the FastAPI demo shape). + + Regression tripwire for per-request full-column channel work: every reply + here must reuse the trace's cached bin-color resolution (LOD doc §2), so + the cycle costs the tier work itself — pyramid compose, one exact re-bin + with its mean-color scan, one drill gather. Re-resolving the channel per + request multiplies this row by the column length and shows up as a + step-function regression.""" + result = benchmark(_colored_drilldown_cycle, colored_drilldown_figure) + assert result > DRILL_N + + def test_stratified_sample_mask(benchmark): """Materialized-id category-stratified sampler used by viewport subsets.""" n = 1_000_000 diff --git a/spec/benchmarks/methodology.md b/spec/benchmarks/methodology.md index 736221f5..12ba7a35 100644 --- a/spec/benchmarks/methodology.md +++ b/spec/benchmarks/methodology.md @@ -123,8 +123,14 @@ reported). 4. `scatter_100M_pan`: pyramid pan/zoom percentiles (post LOD phase 3) — the headline; includes never-blank check (frame sampling: no frame without chart pixels). -5. `drilldown_truth`: CodSpeed native cycle covers density overview → exact - points → density zoom-out; the 10M+ exact-hover oracle remains the larger +5. `drilldown_truth`: CodSpeed native cycles cover density overview → exact + points → density zoom-out, twice — a plain trace and a channel-bearing + twin (`test_adaptive_drilldown_cycle_mean_color`, which adds a mid-band + exact re-bin leg). The twin exists because per-request cost on a colored + trace must exclude full-column channel work: the bin-color resolution is + cached per trace (LOD doc §2), and re-resolving it per reply multiplies + the row by the column length (the 100M FastAPI demo's 1–2 s/request + regression). The 10M+ exact-hover oracle remains the larger browser/widget follow-up. 6. `core_2d_payloads`: CodSpeed tracks native histogram, area, bar, heatmap, and composed/layered `xy.chart(...)` payload prep; @@ -228,7 +234,7 @@ it — those live in `benchmark-refresh.yml`, and the workflow says so inline. The glob collects five modules — `test_codspeed_animation.py`, `test_codspeed_kernels.py`, `test_codspeed_pyplot.py`, -`test_codspeed_selection.py`, and `test_codspeed_transport.py` — for 102 rows +`test_codspeed_selection.py`, and `test_codspeed_transport.py` — for 103 rows total. These are trend-tracked in CodSpeed, not gated: none of them feed `scripts/check_regressions.py`, whose three inputs are §7's. @@ -303,7 +309,7 @@ the path measured rather than the identity fast path. Browser input-to-pixel latency stays in `bench_interaction.py`; before this module a selection regression could only surface as browser wall-clock noise. -**`benchmarks/test_codspeed_kernels.py` — 73 rows** (70 functions; two are +**`benchmarks/test_codspeed_kernels.py` — 74 rows** (71 functions; two are parametrized, over 2 ingest flavors and 3 `bin_2d` thread-cap regimes). This is the bulk of the suite and covers the native compute core the rest of the engine sits on: decimation tiers, f32 encoding, and zone maps (`spec/design-dossier.md` diff --git a/spec/benchmarks/results.md b/spec/benchmarks/results.md index 20112565..2512ba71 100644 --- a/spec/benchmarks/results.md +++ b/spec/benchmarks/results.md @@ -110,9 +110,15 @@ scope, not the kernels module alone. The suite asserts `xy.kernels.BACKEND == payload preparation. - Zoom refresh with `test_m4_indices_zoom` and `test_decimate_view`. - Memory/payload accounting with `test_memory_report_density_medium`. -- The native adaptive drilldown cycle with - `test_adaptive_drilldown_cycle`: a warmed large scatter viewport moves from - density overview, to exact visible points, and back out to density. +- The native adaptive drilldown cycles: `test_adaptive_drilldown_cycle` (a + warmed large scatter viewport moves from density overview, to exact visible + points, and back out to density) and its channel-bearing twin + `test_adaptive_drilldown_cycle_mean_color`, which adds a mid-band exact + re-bin leg and guards the once-per-trace bin-color resolution (LOD doc §2): + re-resolving the color column per request multiplied per-reply cost by + ~10-100× on the 100M FastAPI drilldown demo (measured 1.3-7 s/reply against + 0.02-0.45 s with the trace-cached resolution on a 4-core host, and + 223 ms → 99 ms for the 2.1M benchmark cycle itself). - The `xy.pyplot` shim suite (`benchmarks/test_codspeed_pyplot.py`): paired raw-versus-shim rows for line, scatter, histogram, categorical bar, and styled-panel builds, plus matched PNG export From d78c4d812b66fdb6ffc67d06468eb75c82e228a0 Mon Sep 17 00:00:00 2001 From: Masen Furer Date: Thu, 23 Jul 2026 23:36:38 +0000 Subject: [PATCH 3/5] Bound the colored build's peak memory: chunked quantize, budgeted fan-out MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mean-color feature made a colored huge scatter's ONE-TIME costs scale peak RSS with N-sized temporaries, pushing a 1e9-point colored build past what count-only main needed (main: a stretch; this stack: OOM). Three bounds restore the envelope without changing a single shipped byte: - resolve_bin_colors quantizes full columns in _QUANTIZE_CHUNK slices into a preallocated u8 result. The per-element math is the identical historical chain (normalize f32 -> f64 widen -> x255 -> rint -> u8; bitwise-equal, pinned by test), but the one-shot pipeline's several full-length f64 temporaries — ~20 GB at 1e9 rows, 4 GB at the 205M repro — become ~110 MB of chunk scratch. direct_rgba and wide-code folds chunk the same way. - bin_2d_mean_color_cells sheds workers whenever their private 40 B/cell accumulators would together exceed MEAN_COLOR_ACCUM_BUDGET_BYTES (1 GiB): screen grids and the 2048-square default keep the full 4-way fan-out, a no-rescan trace's adaptive 8192-square base level builds serial — one 2.7 GB accumulator instead of four plus a merge target. The colored build's transient now sits at or below the count-only build's own fan-out. - trace_bin_colors resolves but does not retain for no-rescan traces (memmapped or past PYRAMID_NO_RESCAN_ROWS): after the pyramid exists, every interactive reply composes its prebuilt color planes, so a retained per-row idx (1 GB at 1e9 rows) had no remaining consumer. The rare correctness-net re-bin re-resolves chunk-bounded. Borrowed categorical code arrays also no longer double-count against channel_bytes in the memory report. Measured: the 205M colored first view adds 0.96 GB of peak RSS over the canonical columns instead of 3.96 GB and builds ~35% faster; a 600M memmapped colored drilldown now completes inside a 15 GB host (build stage +1.34 GB anonymous, interactive replies +0.00 GB, correct pyramid-L0-upsampled deep zoom); the 100M demo's warmup halves to 6.5 s with steady-state latencies unchanged. LOD doc §2/§4.4 record the bounds as regressions-to-avoid. --- CHANGELOG.md | 16 ++++++++ python/xy/channels.py | 58 ++++++++++++++++++++++++---- python/xy/interaction.py | 40 +++++++++++++++++--- spec/design/lod-architecture.md | 18 +++++++++ src/kernels.rs | 36 +++++++++++++++++- src/tiles.rs | 8 +++- tests/test_density_mean_color.py | 65 ++++++++++++++++++++++++++++++++ 7 files changed, 226 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6326753..2a906a8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,22 @@ in the README). to the internal engine object. ### Changed +- **Colored huge-scatter builds are peak-memory-bounded (LOD doc §4.4).** + The mean-color feature's one-time costs no longer scale peak RSS with N × + temporaries: the full-column color-source quantize now runs chunked + (bitwise-identical math, transient temporaries bounded by + `channels._QUANTIZE_CHUNK` instead of several × N — ~20 GB at 1e9 rows + before), the colored pyramid's build scan sheds workers so its 40 B/cell + accumulators stay inside a 1 GiB budget (an adaptive 8192² base level + builds serial: one 2.7 GB accumulator, not four), and no-rescan traces + (huge or out-of-core) resolve without retaining the per-row idx — after + the pyramid exists, every interactive reply composes prebuilt color + planes, so retention was resident cost with no consumer. Measured at 205M + rows: the first colored `density_view` now adds 0.96 GB of peak RSS over + the canonical columns instead of 3.96 GB, and builds ~35% faster; a + colored 1e9-point build's transient peak drops to at-or-below the + count-only build's own fan-out, restoring 1B-point capability wherever + count-only main could run. - **Bin-color resolution is resolved once per trace, never per request (LOD doc §2).** `density_view` used to quantize the *entire* color column into the kernel's LUT-index/RGBA source on every reply — an O(N) NumPy pass diff --git a/python/xy/channels.py b/python/xy/channels.py index d5417853..39473715 100644 --- a/python/xy/channels.py +++ b/python/xy/channels.py @@ -561,6 +561,49 @@ def bins_mean_color(cc: Optional[ColorChannel]) -> bool: return cc is not None and cc.mode in ("continuous", "categorical", "direct_rgba") +# Chunk length for full-column color-source quantization. The math is +# element-wise, so chunking changes nothing but the transient footprint: a +# one-shot pipeline materializes several full-length f64 temporaries at once +# (~20 GB at 1e9 rows — the difference between a colored billion-point build +# fitting in RAM or not), while chunked passes keep every temporary at chunk +# size and the only N-sized allocation is the u8 result. +_QUANTIZE_CHUNK = 1 << 22 + + +def _quantized_lut_idx(values: npt.NDArray[np.float64], domain: tuple[float, float]) -> np.ndarray: + """Continuous values -> u8 LUT texel indices, chunk-bounded temporaries. + + Per-element math is exactly the historical one-shot chain — + `normalize_to_unit` (f32), widen to f64, ×255, `rint`, cast u8 — applied + per chunk, so results are bitwise identical while peak memory stays + O(chunk) + the N-byte output.""" + out = np.empty(len(values), dtype=np.uint8) + for start in range(0, len(values), _QUANTIZE_CHUNK): + end = start + _QUANTIZE_CHUNK + unit = normalize_to_unit(values[start:end], domain) + out[start:end] = np.rint(np.asarray(unit, dtype=np.float64) * 255.0).astype(np.uint8) + return out + + +def _quantized_rgba8(values: npt.NDArray[np.float64]) -> np.ndarray: + """Float RGBA rows -> straight-alpha RGBA8, chunk-bounded temporaries.""" + out = np.empty(values.shape, dtype=np.uint8) + for start in range(0, len(values), _QUANTIZE_CHUNK): + end = start + _QUANTIZE_CHUNK + seg = values[start:end] + out[start:end] = np.rint(np.clip(seg, 0.0, 1.0) * 255.0).astype(np.uint8) + return out + + +def _folded_codes_u8(codes: np.ndarray, n_palette: int) -> np.ndarray: + """Wide categorical codes -> u8 palette rows (mod fold), chunk-bounded.""" + out = np.empty(len(codes), dtype=np.uint8) + for start in range(0, len(codes), _QUANTIZE_CHUNK): + end = start + _QUANTIZE_CHUNK + out[start:end] = (codes[start:end] % n_palette).astype(np.uint8) + return out + + def resolve_bin_colors(cc: Optional[ColorChannel], sel: Any, palette: list[str]) -> Optional[dict]: """Kernel color source for mean-color density binning (LOD doc §2). @@ -579,7 +622,7 @@ def resolve_bin_colors(cc: Optional[ColorChannel], sel: Any, palette: list[str]) if rgba is None: raise ValueError("direct RGBA color channel missing values") values = rgba if sel is None else rgba[sel] - return {"rgba": np.rint(np.clip(values, 0.0, 1.0) * 255.0).astype(np.uint8)} + return {"rgba": _quantized_rgba8(values)} if cc.mode == "continuous": values = cc.values domain = cc.domain @@ -587,10 +630,9 @@ def resolve_bin_colors(cc: Optional[ColorChannel], sel: Any, palette: list[str]) raise ValueError("continuous color channel missing values or domain") vals = values if sel is None else values[sel] # Same normalization the wire ships, quantized to the nearest of the - # client's 256 LUT texels. - unit = normalize_to_unit(vals, domain) - idx = np.rint(np.asarray(unit, dtype=np.float64) * 255.0).astype(np.uint8) - return {"idx": idx, "lut": colormap_lut_rgba8(cc.colormap)} + # client's 256 LUT texels (chunked: full-column calls keep transient + # temporaries chunk-bounded instead of several × N). + return {"idx": _quantized_lut_idx(vals, domain), "lut": colormap_lut_rgba8(cc.colormap)} code_values = cc.codes categories = cc.categories if code_values is None or categories is None: @@ -601,8 +643,10 @@ def resolve_bin_colors(cc: Optional[ColorChannel], sel: Any, palette: list[str]) # >256 categories ship wide codes; palette colors repeat every # len(palette) categories, so folding the codes onto the base palette # bins each point with exactly the color it draws with. - folded = (codes % len(palette)).astype(np.uint8) - return {"idx": folded, "lut": palette_rgba8(palette, len(palette))} + return { + "idx": _folded_codes_u8(codes, len(palette)), + "lut": palette_rgba8(palette, len(palette)), + } def ship_channels( diff --git a/python/xy/interaction.py b/python/xy/interaction.py index d19b903d..dc43875e 100644 --- a/python/xy/interaction.py +++ b/python/xy/interaction.py @@ -399,23 +399,53 @@ def trace_bin_colors(t: Trace) -> Optional[dict]: no mean-color grid (pyramid replies use the prebuilt color planes, drills ship sliced channels). A rebuildable derived cache (§27), dropped by `append_data` (appended rows' colors and a possibly moved channel domain - both change the resolution).""" + both change the resolution). + + Huge/out-of-core traces (the §28 no-rescan regime) resolve but do NOT + retain: past the one-time colored-pyramid build and first-paint emit, + every interactive reply composes the pyramid's prebuilt color planes, so + a retained per-row idx (1 GB at 1e9 rows) would be resident cost with no + remaining consumer. The rare correctness-net re-bin re-resolves — chunked + (`channels._QUANTIZE_CHUNK`), so the recompute is CPU, not a memory + spike.""" cached = t._bin_colors if cached is None: resolved = channels.resolve_bin_colors(t.color_ch, None, DEFAULT_PALETTE) - t._bin_colors = 0 if resolved is None else resolved + if not (is_memmapped(t.x.values) or len(t.x) > PYRAMID_NO_RESCAN_ROWS): + t._bin_colors = 0 if resolved is None else resolved return resolved return None if cached == 0 else cached def bin_color_cache_bytes(fig: Any) -> int: """Memory-report line (design dossier §27): bytes held by resolved - bin-color caches (per-row idx/rgba planes plus their LUTs).""" + bin-color caches (per-row idx/rgba planes plus their LUTs). + + Arrays the resolution merely borrows from the channel's own storage + (compact u8 categorical codes pass through by reference) are already + counted as `channel_bytes` — skip them so `resident_array_bytes` never + double-counts one buffer.""" total = 0 for t in fig.traces: cached = getattr(t, "_bin_colors", None) - if cached: - total += sum(int(v.nbytes) for v in cached.values() if isinstance(v, np.ndarray)) + if not cached: + continue + cc = t.color_ch + owned = [ + arr + for arr in ( + getattr(cc, "codes", None), + getattr(cc, "values", None), + getattr(cc, "rgba", None), + ) + if arr is not None + ] + for v in cached.values(): + if not isinstance(v, np.ndarray): + continue + if any(np.shares_memory(v, arr) for arr in owned): + continue + total += int(v.nbytes) return total diff --git a/spec/design/lod-architecture.md b/spec/design/lod-architecture.md index 80592a43..7deebdf7 100644 --- a/spec/design/lod-architecture.md +++ b/spec/design/lod-architecture.md @@ -151,6 +151,12 @@ lightness (the points' own alpha compositing). channels, so a per-request or per-tier resolve is a regression (it charged the 100M FastAPI drilldown demo 1–2 s per reply, ~10–100× the tier work; tripwire: `test_adaptive_drilldown_cycle_mean_color`). + Huge/out-of-core traces (the §28 no-rescan regime) resolve but do **not** + retain: past the one-time pyramid build and first-paint emit their every + interactive reply composes prebuilt planes, so a retained per-row idx + (1 GB at 1e9 rows) would be resident cost with no remaining consumer. + The resolution itself is chunk-bounded (§4.4), so the rare + correctness-net re-resolve costs CPU, never a memory spike. *Why mean-of-colors and not colormap(mean value):* the mean color is representation-agnostic (categories have no mean value), matches the drilled points' downsample exactly (the anti-jarring contract, T3), and @@ -285,6 +291,18 @@ ever extrapolates. 45 MB color per colored trace, `pyramid_report_bytes`; huge/out-of-core traces build adaptively finer bases — Phase-3 item 7 — and the color plane scales with them.) +- **Build-time transients are budgeted, not incidental.** The colored build's + fused scan gives each worker a private 40 B/cell accumulator; workers shed + (down to a serial scan) whenever their accumulators together would exceed + a fixed budget (`kernels::MEAN_COLOR_ACCUM_BUDGET_BYTES`, 1 GiB), so an + adaptive 8192² base level costs one 2.7 GB accumulator instead of 4×, and + a colored billion-point build peaks at or below the count-only build's own + fan-out. The full-column color-source resolution feeding it is chunked + (`channels._QUANTIZE_CHUNK`): per-element math identical to the one-shot + chain, transient temporaries bounded by the chunk instead of several × N + (~20 GB at 1e9 rows before this), the only N-sized allocation the u8 idx + itself. Both are recorded regressions-to-avoid: trading them back returns + the 1B colored build to OOM territory. - 1B points: ~330-660 MB — still kernel-side RAM, but now Tier 3 applies: tiles are chunked to disk (Arrow/Parquet row groups per tile, dossier §32), LRU-resident under a byte budget, and *only* the ≤ ~12 visible tiles are diff --git a/src/kernels.rs b/src/kernels.rs index 6551e6b4..7e778516 100644 --- a/src/kernels.rs +++ b/src/kernels.rs @@ -3441,6 +3441,27 @@ impl MeanColorCell { /// pyramid's 100M-row base scan to a quarter. Integer sums merge /// order-independently — bitwise deterministic for any thread count. #[allow(clippy::too_many_arguments)] +/// Byte budget for the mean-color scan's worker accumulators together. Each +/// worker owns a private `cells × size_of::()` (40 B/cell) +/// grid; screen-sized grids and the 2048² default base level fit the full +/// 4-worker fan-out comfortably, but a no-rescan trace's adaptive base level +/// (§28; up to 16384² cells) would turn that fan-out into tens of GB of +/// transient build memory — the difference between a colored billion-point +/// build fitting in RAM or not. Workers over budget are shed down to a +/// serial scan: one-time build wall-clock is traded, peak RSS never. (The +/// merge target grid rides on top of the budget; at every size where memory +/// binds, the scan is already serial and allocates exactly one grid.) +const MEAN_COLOR_ACCUM_BUDGET_BYTES: usize = 1 << 30; + +/// Pure fan-out choice for the mean-color scan: the row-scan fan-out +/// (`bin_2d_threads`) capped at 4, then shed to whatever worker count keeps +/// the private accumulators inside `MEAN_COLOR_ACCUM_BUDGET_BYTES`. +fn mean_color_threads_for(row_threads: usize, cells: usize) -> usize { + let per_grid = cells.saturating_mul(std::mem::size_of::()); + let by_memory = (MEAN_COLOR_ACCUM_BUDGET_BYTES / per_grid.max(1)).max(1); + row_threads.min(4).min(by_memory) +} + pub(crate) fn bin_2d_mean_color_cells( x: &[f64], y: &[f64], @@ -3454,7 +3475,7 @@ pub(crate) fn bin_2d_mean_color_cells( ) -> Vec { let n = x.len(); let cells = w * h; - let threads = bin_2d_threads(n, cells).min(4); + let threads = mean_color_threads_for(bin_2d_threads(n, cells), cells); if threads <= 1 || n < threads { let mut grid = vec![MeanColorCell::default(); cells]; bin_2d_mean_color_accumulate(x, y, colors, 0, x0, x1, y0, y1, w, h, &mut grid); @@ -6416,6 +6437,19 @@ mod tests { } } + #[test] + fn mean_color_threads_shed_to_fit_accumulator_budget() { + // Screen-sized grids and the 2048² default base level keep the full + // 4-worker fan-out; the adaptive no-rescan base levels (§28) shed to + // serial so worker accumulators (40 B/cell) never multiply into + // tens of GB of transient build memory. + assert_eq!(mean_color_threads_for(16, 512 * 384), 4); + assert_eq!(mean_color_threads_for(4, 2048 * 2048), 4); + assert_eq!(mean_color_threads_for(4, 4096 * 4096), 1); + assert_eq!(mean_color_threads_for(4, 16384 * 16384), 1); + assert_eq!(mean_color_threads_for(1, 64), 1); + } + #[test] fn bin_2d_density_hotspot() { // A cluster in one cell should dominate grid_max. diff --git a/src/tiles.rs b/src/tiles.rs index b4c9f809..30a54585 100644 --- a/src/tiles.rs +++ b/src/tiles.rs @@ -102,8 +102,12 @@ pub fn build( /// sums together (exact integer sums merged order-independently, so the /// result is bitwise identical for any fan-out); count levels are identical /// to `build`'s. The scan fans out only when rows outnumber base cells -/// (points-per-cell gate, capped at 4 workers): each worker's accumulator is -/// 40 B/cell (~170 MB at the 2048² default), released before returning — +/// (points-per-cell gate, capped at 4 workers, and shed further so the +/// workers' 40 B/cell accumulators stay inside a fixed byte budget — +/// `kernels::MEAN_COLOR_ACCUM_BUDGET_BYTES`): ~170 MB per worker at the +/// 2048² default keeps the full fan-out, while a no-rescan trace's adaptive +/// base level (up to 16384²) builds serial rather than multiplying GB-scale +/// accumulators by the worker count. All are released before returning — /// builds are one-time per trace and lazy. #[allow(clippy::too_many_arguments)] pub fn build_color( diff --git a/tests/test_density_mean_color.py b/tests/test_density_mean_color.py index db014d9c..bdc16d8e 100644 --- a/tests/test_density_mean_color.py +++ b/tests/test_density_mean_color.py @@ -435,6 +435,71 @@ def test_append_re_resolves_bin_colors_exactly_once(monkeypatch): assert calls["n"] == 2 +def test_chunked_quantization_matches_one_shot_chain(monkeypatch): + # The chunked resolve exists purely to bound transient memory (a one-shot + # chain materializes several full-length f64 temporaries — ~20 GB at 1e9 + # rows); per-element math must stay bitwise identical to the historical + # pipeline, including non-finite and out-of-domain values and chunk + # boundaries. + monkeypatch.setattr(channels, "_QUANTIZE_CHUNK", 7) + rng = np.random.default_rng(16) + vals = rng.uniform(-5.0, 15.0, 1000) + vals[::17] = np.nan + vals[3::29] = np.inf + vals[5::31] = -np.inf + domain = (0.0, 10.0) + unit = channels.normalize_to_unit(vals, domain) + want = np.rint(np.asarray(unit, dtype=np.float64) * 255.0).astype(np.uint8) + assert np.array_equal(channels._quantized_lut_idx(vals, domain), want) + + rgba = rng.uniform(-0.2, 1.2, (500, 4)) + want_rgba = np.rint(np.clip(rgba, 0.0, 1.0) * 255.0).astype(np.uint8) + assert np.array_equal(channels._quantized_rgba8(rgba), want_rgba) + + codes = rng.integers(0, 300, 400).astype(np.uint32) + n_palette = len(DEFAULT_PALETTE) + want_codes = (codes % n_palette).astype(np.uint8) + assert np.array_equal(channels._folded_codes_u8(codes, n_palette), want_codes) + + +def test_no_rescan_traces_resolve_without_retention(monkeypatch): + # Past the no-rescan threshold every interactive reply composes prebuilt + # pyramid planes, so retaining a per-row idx (1 GB at 1e9 rows) would be + # resident cost with no consumer: resolve on demand, never store. + from xy import interaction + + calls = _count_resolves(monkeypatch) + n = SCATTER_DENSITY_THRESHOLD * 3 + monkeypatch.setattr(interaction, "PYRAMID_NO_RESCAN_ROWS", n - 1) + rng = np.random.default_rng(17) + x = rng.uniform(0.0, 100.0, n) + y = rng.uniform(0.0, 100.0, n) + fig = Figure().scatter(x, y, color=x.copy(), density=True) + upd, _ = fig.density_view(0, 0.0, 100.0, 0.0, 100.0, 256, 192) + tr = upd["traces"][0] + assert tr["binning"] == "bin2d-oversized" # the no-rescan correctness net + assert tr["density"].get("color_agg") == "mean" # the plane still ships + assert calls["n"] == 1 + assert fig.traces[0]._bin_colors is None # resolved, not retained + fig.density_view(0, 0.0, 100.0, 0.0, 100.0, 256, 192) + assert calls["n"] == 2 # re-resolved (chunk-bounded), by design + assert fig.memory_report()["bin_color_bytes"] == 0 + + +def test_categorical_cache_counts_only_owned_arrays(): + # Compact u8 categorical codes pass through the resolution by reference; + # they are already counted as channel_bytes, so the cache line must count + # only what the resolution owns (the palette LUT). + n = SCATTER_DENSITY_THRESHOLD + 10_000 + rng = np.random.default_rng(18) + x = rng.uniform(0.0, 10.0, n) + cats = np.where(x < 5.0, "left", "right") + fig = Figure().scatter(x, rng.uniform(0.0, 1.0, n), color=cats, density=True) + report = fig.memory_report() + assert fig.traces[0]._bin_colors["idx"] is fig.traces[0].color_ch.codes + assert report["bin_color_bytes"] == 2 * 4 # the 2-row RGBA8 palette LUT only + + def test_memory_report_itemizes_bin_color_cache_bytes(): n = SCATTER_DENSITY_THRESHOLD * 3 rng = np.random.default_rng(15) From 5855b768d45bfc734648db00af565a7548566b18 Mon Sep 17 00:00:00 2001 From: Masen Furer Date: Fri, 24 Jul 2026 00:28:12 +0000 Subject: [PATCH 4/5] Reattach the too_many_arguments allow to bin_2d_mean_color_cells MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The accumulator-budget commit inserted the budget const and the fan-out helper between the function's doc comment and its #[allow(clippy::too_many_arguments)], leaving the attribute (and the doc) attached to the const and the 9-argument function bare — clippy -D warnings failed. Restore the doc + allow onto the function and fold the budget note into its fan-out paragraph. --- src/kernels.rs | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/kernels.rs b/src/kernels.rs index 7e778516..339e99e8 100644 --- a/src/kernels.rs +++ b/src/kernels.rs @@ -3430,17 +3430,6 @@ impl MeanColorCell { } } -/// Fused count+color accumulation over a full grid: the shared core of -/// `bin_2d_mean_color` and the pyramid build's base pass -/// (`tiles::build_color`). Returns the raw accumulator cells so callers keep -/// exact counts alongside the color means. -/// -/// Fan-out is gated by the points-per-cell ratio (like `bin_2d`) and capped -/// at 4: each worker's private accumulator is ~10× a count grid (40 B/cell), -/// so the cap bounds the transient to ~4 grids while still cutting the -/// pyramid's 100M-row base scan to a quarter. Integer sums merge -/// order-independently — bitwise deterministic for any thread count. -#[allow(clippy::too_many_arguments)] /// Byte budget for the mean-color scan's worker accumulators together. Each /// worker owns a private `cells × size_of::()` (40 B/cell) /// grid; screen-sized grids and the 2048² default base level fit the full @@ -3462,6 +3451,18 @@ fn mean_color_threads_for(row_threads: usize, cells: usize) -> usize { row_threads.min(4).min(by_memory) } +/// Fused count+color accumulation over a full grid: the shared core of +/// `bin_2d_mean_color` and the pyramid build's base pass +/// (`tiles::build_color`). Returns the raw accumulator cells so callers keep +/// exact counts alongside the color means. +/// +/// Fan-out is gated by the points-per-cell ratio (like `bin_2d`), capped at +/// 4, and shed further to fit `MEAN_COLOR_ACCUM_BUDGET_BYTES`: each worker's +/// private accumulator is ~10× a count grid (40 B/cell), so the cap bounds +/// the transient while still cutting the pyramid's 100M-row base scan to a +/// quarter. Integer sums merge order-independently — bitwise deterministic +/// for any thread count. +#[allow(clippy::too_many_arguments)] pub(crate) fn bin_2d_mean_color_cells( x: &[f64], y: &[f64], From 02347481fad4ca24d3127e7ceb8971aa630dfea2 Mon Sep 17 00:00:00 2001 From: Masen Furer Date: Fri, 24 Jul 2026 01:31:53 +0000 Subject: [PATCH 5/5] Name the aggregate floor on the drilldown badge; note the >200M drill gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Past PYRAMID_NO_RESCAN_ROWS (200M rows, or any disk-backed column) the engine serves every zoom from the pyramid — upsampled at its floor — and never ships exact points: the O(N) window rescan a drill needs is forbidden in that regime (LOD doc §28), however deep the window. Run the demo with XY_LIVE_POINTS above the bound and that contract reads as a drilldown that mysteriously refuses to resolve; the reply has recorded the decision all along (binning: pyramid-L0-upsampled), the page just never showed it. The badge now appends 'aggregate floor' (with a tooltip naming the regime) whenever the reply's binning says the aggregate is served past its resolution, and the README documents the scale gate next to the XY_LIVE_POINTS knob it rides on. --- examples/fastapi/README.md | 8 ++++++++ examples/fastapi/live_drilldown.py | 18 +++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/examples/fastapi/README.md b/examples/fastapi/README.md index e9023568..0cbd2904 100644 --- a/examples/fastapi/README.md +++ b/examples/fastapi/README.md @@ -38,6 +38,14 @@ on first use: XY_LIVE_POINTS=1000000 uv run uvicorn app:app ``` +Note the drill-to-points behavior is scale-gated by the engine +(`xy.config.PYRAMID_NO_RESCAN_ROWS`, 200M): above that row count — or for +disk-backed columns — every zoom is answered from the density pyramid, +upsampled at its floor, and exact points never ship (the O(N) window rescan +that drilling requires is forbidden in that regime; LOD doc §28). The badge +reads `… density · aggregate floor` when a view is in that state, so a 1B-row +run that "never reaches points" is the recorded contract, not a bug. + ## Layout | File | Role | diff --git a/examples/fastapi/live_drilldown.py b/examples/fastapi/live_drilldown.py index 79860bbd..e8aa1034 100644 --- a/examples/fastapi/live_drilldown.py +++ b/examples/fastapi/live_drilldown.py @@ -294,7 +294,23 @@ def live_drilldown_html() -> str: const trace = message.traces && message.traces[0]; if (trace && trace.mode && message.seq === view.seq) {{ const visible = Number(trace.visible || 0).toLocaleString(); - statusEl.textContent = trace.mode === "points" ? `${{visible}} points` : `${{visible}} density`; + if (trace.mode === "points") {{ + statusEl.textContent = `${{visible}} points`; + statusEl.title = ""; + }} else {{ + // Surface the engine's recorded tier decision (§28): past + // PYRAMID_NO_RESCAN_ROWS (or for out-of-core columns) every zoom is + // served from the pyramid — upsampled at its floor — and exact + // points are unavailable BY DESIGN, however deep the window. Without + // this note that regime reads as a drilldown that mysteriously + // refuses to resolve. + const floored = String(trace.binning || "").endsWith("-upsampled"); + statusEl.textContent = `${{visible}} density${{floored ? " · aggregate floor" : ""}}`; + statusEl.title = floored + ? "No-rescan regime (source rows exceed the exact-rescan bound): " + + "zooms are served from the density pyramid and exact points never ship." + : ""; + }} }} }} catch (err) {{ statusEl.textContent = "offline";