diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f75630f..2b136839 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,69 @@ in the README). to the internal engine object. ### Changed +- **No sampled points above the resolution of the graph (#225).** Interactive + `density_view` replies no longer ship a point-sample overlay: a fixed-size + sample above the drill budget reads as individual data points at a zoom + where real points are sub-pixel, misrepresenting the dataset — the + mean-color density surface stands alone there. The retained first-payload + sample (also the standalone re-bin worker's CPU source) now draws only when + the view's estimated in-view count fits the direct point budget, i.e. when + individual points are actually resolvable; real points still ship the + moment a window fits the budget, so drilldown behavior is unchanged. +- **Mean-color density surfaces composite like the points they aggregate + (LOD doc §2).** A channel-bearing cell's displayed alpha is now the + physical compositing of its own points — `1 − (1 − ā)^count` for mean + point alpha ā — instead of a per-window log-count tone curve. Lightness + no longer swings between windows or across the texture↔points boundary + (the aggregate is exactly as saturated as overplotted real marks), the + texture upload is normalization-free (no exposure re-uploads), and + mean-color drills swap at native opacity with no intensity handoff. + Count-only (constant-color) surfaces keep the log ramp — count is their + only structure. Same law in the client, the SVG/PNG exporters, and the + standalone re-bin worker. +- **The aggregate tier no longer refines; density requests only probe the + points band (LOD doc T13).** Whatever density texture already covers the + view stands — however blurry — until the estimated in-view count comes + within `LOD_POINTS_REQUEST_BAND ×` the direct budget; only then does a + `density_view` go out, and the kernel answers with exact points once the + count fits. The estimate takes the lower of an area-scaled cached-window + count and the retained first-payload sample counted in-view — the sample + follows the data's actual distribution, so sparse regions reach their + points without being stranded in blur by uniform-density assumptions. + Display-side, the aggregate sharpens in QUANTIZED ladder steps between + home and points (`LOD_AGG_STEP_FACTOR`/`LOD_AGG_STEP_MAX`: the view + snapped outward to a power-of-4 block grid over the extent, at most two + steps) — pan-stable, dedupable windows, so a zoom sees at most two + smooth-to-smooth texture swaps and worst-case softness is bounded at + ~4× stretch per axis. A step reply is the only density reply that may + repaint a covered view; mid-band probe replies (the band's exact grids + have a speckled character that read as zoom-level jumping against the + smooth standing surface) land as facts-only cache entries for the gate. + Replies for uncovered views still apply, and standalone clients keep + applying everything. A 100M-scatter field capture had shipped a ~2.7 MB full-screen + grid on every pan/zoom step (including sub-pixel window twins, now deduped + within half an output texel) for what was the same aggregate with + marginally different blur; intermediate-zoom blur is the accepted, + recorded cost. Transition-band replies that do go out are clamped to the + pyramid's source resolution — never more cells than the finest level + resolves under the window. Kernel-attached clients also no longer draw the + retained first-payload sample at any zoom (resolvable views get real + points, not retained sample rows); it remains the standalone client's + fallback, gated by resolvability. A tried fine-over-broad texture layering + was reverted (recorded): density textures alpha-composite, so overlaps + double-count opacity, and per-window normalization makes the seam a + brightness step. +- **Full-point windows are padded, aligned, cached, and never re-requested + (LOD doc T13).** A points-tier reply now ships the largest aligned window + around the view whose exact count still fits the budget (bounds snapped to + a power-of-two grid over the trace's extent, per dimension), so consecutive + pans resolve to the same window; the client retires replaced exact windows + into a bounded per-trace cache and promotes them back — pan ping-pong and + zoom-out/zoom-in render entirely from the GPU with zero round-trips. Picks + against a promoted (older) window still resolve exactly through a bounded + kernel-side subset history. Identical `density_view` requests (same window, + same screen, unchanged data) are suppressed client-side, and a suppressed + duplicate's in-flight reply is accepted instead of dying to the seq race. - **Density surfaces now wear the data's own colors (LOD doc §2).** A Tier-2 scatter's aggregated view colors each cell with the alpha-weighted **mean of its binned points' resolved colors** (continuous colormap, categorical diff --git a/benchmarks/test_codspeed_kernels.py b/benchmarks/test_codspeed_kernels.py index c5f207fc..5eee704a 100644 --- a/benchmarks/test_codspeed_kernels.py +++ b/benchmarks/test_codspeed_kernels.py @@ -231,13 +231,26 @@ def drilldown_figure() -> Figure: x = rng.uniform(0.0, 100.0, DRILL_N).astype(np.float64, copy=False) y = rng.uniform(0.0, 100.0, DRILL_N).astype(np.float64, copy=False) fig = xy.chart(xy.scatter(x=x, y=y, density=True)).figure() - fig._benchmark_deep_expected = int(np.count_nonzero((x < 10.0) & (y < 10.0))) # Warm the lazily-built pyramid so CodSpeed tracks interactive viewport # refresh cost, not one-time index construction. fig.density_view(0, 0.0, 100.0, 0.0, 100.0, GRID_W, GRID_H) - fig.density_view(0, 0.0, 10.0, 0.0, 10.0, GRID_W, GRID_H) + deep, _ = fig.density_view(0, 0.0, 10.0, 0.0, 10.0, GRID_W, GRID_H) fig.density_view(0, 0.0, 100.0, 0.0, 100.0, GRID_W, GRID_H) + + # The drill ships the widest ALIGNED window around the view that still + # fits the budget (LOD doc T13), so the count oracle is the SHIPPED + # window's, taken from the warm-up reply — alignment is deterministic, + # every benchmark iteration resolves to the same window. + trace = deep["traces"][0] + assert trace["mode"] == "points" + (wx0, wx1), (wy0, wy1) = trace["x_range"], trace["y_range"] + assert wx0 <= 0.0 and wx1 >= 10.0 and wy0 <= 0.0 and wy1 >= 10.0 + fig._benchmark_deep_window = (wx0, wx1, wy0, wy1) + fig._benchmark_deep_expected = int( + np.count_nonzero((x >= wx0) & (x <= wx1) & (y >= wy0) & (y <= wy1)) + ) + assert fig._benchmark_deep_expected == trace["visible"] return fig @@ -1068,6 +1081,10 @@ def _adaptive_drilldown_cycle(fig: Figure) -> int: assert wide_trace["mode"] == "density" assert deep_trace["mode"] == "points" assert str(wide_trace.get("binning", "")).startswith("pyramid-L") + # T13: the shipped window is the deterministic padded aligned superset of + # the view, so both the window and its exact count repeat every cycle. + assert tuple(deep_trace["x_range"]) == fig._benchmark_deep_window[:2] + assert tuple(deep_trace["y_range"]) == fig._benchmark_deep_window[2:] assert deep_trace["visible"] == fig._benchmark_deep_expected drill_seq = deep_trace["drill_seq"] row = fig.pick(0, 0, drill_seq) diff --git a/docs/core-concepts/large-data-and-performance.md b/docs/core-concepts/large-data-and-performance.md index e6c4d8c1..119ca434 100644 --- a/docs/core-concepts/large-data-and-performance.md +++ b/docs/core-concepts/large-data-and-performance.md @@ -16,8 +16,8 @@ readout and refinement. | --- | --- | --- | | Direct | Small visible traces | One rendered point or segment per retained row | | M4-decimated | Long ordered lines and areas | A bounded sequence preserving per-bucket extrema | -| Density + sample | Dense scatter overviews | A fixed-resolution count grid plus a deterministic point sample | -| Refined view | A narrower pan/zoom window | A new viewport-specific aggregate or exact visible points when they fit | +| Density | Dense scatter overviews | A count grid wearing the data's own mean point colors. It stands across pans and zooms — softening as you go deeper — rather than re-aggregating per viewport; in standalone (kernel-less) charts a deterministic point sample overlays it once the view's estimated count would fit the direct budget | +| Refined view | A window near or below the direct budget | Exact visible points for a padded aligned window — nearby pans and zooms then render from the cached window with no further requests | The current defaults begin M4 line decimation above 10,000 rows and automatic scatter density above 200,000 points. Density grids default to 512×384 cells, diff --git a/js/src/45_lod.ts b/js/src/45_lod.ts index caf2d350..8d5bab9f 100644 --- a/js/src/45_lod.ts +++ b/js/src/45_lod.ts @@ -4,6 +4,10 @@ import { parseColor } from "./20_theme"; const LOD_DIRECT_POINT_BUDGET = 200000; const LOD_DRILL_EXIT_FACTOR = 1.15; +// Retired exact point windows kept per trace (T13), beyond the live drill. +// Each holds ≤ budget points of GPU buffers, so the cap bounds VRAM; the +// outgrown sweep (T11 rule) frees entries geometry can no longer revive. +const LOD_POINT_CACHE_WINDOWS = 3; // Retained-sample zoom-out fade band (T9): full alpha while the sample's home // window still covers ≥ HI of the view area, gone below LO (log-eased between, // applied as composited opacity — see lodSampleViewAlpha). @@ -50,37 +54,58 @@ export function lodCopyGrid(f32) { return f32.slice ? f32.slice() : new Float32Array(f32); } -// Log tone-mapped grid upload: stable perception across renormalization, and -// the u_max swings between rebins compress logarithmically (§5/§F6). + +// Grid upload. Count-only grids upload as R8 log tone-mapped (the shader +// tints/LUTs them): count is their only structure, and the u_max swings +// between rebins compress logarithmically (§5/§F6). // -// Count-only grids upload as R8 (the shader tints/LUTs them). Mean-color -// grids (LOD doc §2) pass the straight-alpha RGBA plane shipped by the -// kernel: rgb = per-cell mean point color, a = mean point alpha. The texture -// bakes the count tone curve into the alpha and stores rgb PREMULTIPLIED — -// in sRGB space, exactly like the mark shaders' outputs — so bilinear -// filtering weights color by coverage instead of dragging occupied cells -// toward transparent-black neighbors. Exposure easing re-calls this per -// norm step; rgb is re-premultiplied against the eased alpha each time. +// Mean-color grids (LOD doc §2) pass the straight-alpha RGBA plane shipped +// by the kernel: rgb = per-cell mean point color, a = mean point alpha. +// Their displayed alpha is the PHYSICAL compositing of the cell's own +// points — `1 − (1 − a_pt)^k` for k points whose drawn per-point alpha is +// a_pt = channel alpha × the trace's style opacity (`pointAlpha`) — exactly +// what the eye would see if every point were drawn sub-pixel, so the +// texture and real marks agree on lightness at every zoom and the +// texture↔points swap never shifts tone (the old window-normalized +// log-count curve made lightness swing across that boundary and between +// windows). Style opacity folds INSIDE the exponent — dense cells saturate +// past it exactly like overplotted marks do — so the draw path applies only +// transition fades on top of these textures, never the style opacity again. +// Count's display role reduces to occupancy and the compositing exponent; +// no per-window normalization touches these textures (max-independent +// upload). rgb stores PREMULTIPLIED — in sRGB space, exactly like the mark +// shaders' outputs — so bilinear filtering weights color by coverage +// instead of dragging occupied cells toward transparent-black neighbors. // `filter` picks the sampling the reply asked for (spatial-exact grids ship // "nearest"); it applies to both texture layouts. -export function lodWriteGridTexture(gl, tex, f32, w, h, maxVal, rgba = null, filter = "linear") { +export function lodWriteGridTexture( + gl, tex, f32, w, h, maxVal, rgba = null, filter = "linear", pointAlpha = 1, +) { const denom = Math.log1p(Math.max(0, maxVal || 0)); let data; if (rgba) { data = new Uint8Array(f32.length * 4); - if (denom > 0) { - for (let i = 0; i < f32.length; i++) { - const c = f32[i]; - if (!(c > 0) || !Number.isFinite(c)) continue; - const t = Math.min(1, Math.log1p(c) / denom); - const shaped = Math.min(1, t * 1.35) * (rgba[i * 4 + 3] / 255); - if (shaped <= 0) continue; - const a = Math.max(1, Math.round(255 * shaped)); - data[i * 4] = Math.round(rgba[i * 4] * a / 255); - data[i * 4 + 1] = Math.round(rgba[i * 4 + 1] * a / 255); - data[i * 4 + 2] = Math.round(rgba[i * 4 + 2] * a / 255); - data[i * 4 + 3] = a; - } + // ln(1 − a_pt) per possible channel-alpha byte, so the per-cell work is + // one exp(). NaN marks saturation (a_pt == 1 → any occupied cell is 1). + const op = Math.min(1, Math.max(0, pointAlpha)); + const logOneMinus = new Float64Array(256); + for (let a8 = 0; a8 < 256; a8++) { + const aPt = (a8 / 255) * op; + logOneMinus[a8] = aPt >= 1 ? NaN : Math.log1p(-aPt); + } + for (let i = 0; i < f32.length; i++) { + const c = f32[i]; + if (!(c > 0) || !Number.isFinite(c)) continue; + const a8 = rgba[i * 4 + 3]; + if (a8 === 0) continue; // an all-invisible cell never invents coverage + const lg = logOneMinus[a8]; + const shaped = Number.isNaN(lg) ? 1 : 1 - Math.exp(c * lg); + if (shaped <= 0) continue; + const a = Math.max(1, Math.round(255 * shaped)); + data[i * 4] = Math.round(rgba[i * 4] * a / 255); + data[i * 4 + 1] = Math.round(rgba[i * 4 + 1] * a / 255); + data[i * 4 + 2] = Math.round(rgba[i * 4 + 2] * a / 255); + data[i * 4 + 3] = a; } } else { data = new Uint8Array(f32.length); @@ -141,7 +166,7 @@ function lodStartNormAnim(view, g, start, target) { g.densityNormMax = target; lodWriteGridTexture( view.gl, g.density.tex, g.density.grid, g.density.w, g.density.h, target, - g.density.rgba, g.density.filter, + g.density.rgba, g.density.filter, view._fillOpacity(g.trace.style), ); return; } @@ -165,7 +190,10 @@ function lodStepNorm(view, g) { if (rel > 0.004 || t >= 1) { d.normMax = norm; g.densityNormMax = norm; - lodWriteGridTexture(view.gl, d.tex, d.grid, d.w, d.h, norm, d.rgba, d.filter); + lodWriteGridTexture( + view.gl, d.tex, d.grid, d.w, d.h, norm, d.rgba, d.filter, + view._fillOpacity(g.trace.style), + ); } if (t < 1) { view.draw(); @@ -250,6 +278,28 @@ function lodSampleViewAlpha(view, s) { return k > 1 ? 1 - Math.pow(1 - band, 1 / k) : band; } +// #225 resolvability gate: a sample overlay draws only when the view it +// would describe could plausibly be points-tier — the estimated in-view +// count (the overlay's recorded window count, scaled by the view's share of +// its window) fits the direct budget. Above that, a fixed-size sample reads +// as individual data points at a zoom where real points are sub-pixel: +// sampling above the resolution of the graph misrepresents the dataset, and +// the aggregate surface — which wears the data's own colors (LOD doc §2) — +// is the truthful representation. In kernel mode the gate makes the hybrid +// look transient at most (real points ship the moment a window fits the +// budget); standalone exports keep the overlay as their only point +// representation once a zoom resolves it. Overlays without recorded counts +// (hand-built/legacy specs) keep drawing — the gate needs facts, not guesses. +function lodOverlayResolvable(view, o) { + const meta = o.sample; + if (!meta || !Number.isFinite(meta.visible)) return true; + const v = view.view; + const viewArea = Math.abs((v.x1 - v.x0) * (v.y1 - v.y0)); + const winArea = lodWindowArea(o.win); + const share = winArea > 0 && viewArea > 0 ? Math.min(1, viewArea / winArea) : 1; + return meta.visible * share <= LOD_DIRECT_POINT_BUDGET; +} + // §28 hybrid overlay, window pairing (T9): every sample rides the density // window it was computed for, so the points on screen always describe the // window being displayed. Selection mirrors lodDensityForView: the smallest @@ -258,7 +308,17 @@ function lodSampleViewAlpha(view, s) { // of a stale drilled cluster). Only when NO cached window covers the view // (pan off-cache, zoom-out past home) does the best partial overlay draw, // bounded by the T9 coverage fade so it can never read as a false cluster. +// Every candidate passes the #225 resolvability gate first: above the direct +// budget the aggregate stands alone. export function lodSampleForView(view, g) { + // Kernel-attached clients never draw retained samples (#225 field + // follow-up): whenever a view is resolvable the kernel answers with REAL + // points — the retained sample is a handful of arbitrary rows drawn at + // full alpha exactly where exact marks are one request away (or already + // on screen), which reads as data that isn't there. The overlay exists + // for the standalone (kernel-less) client, whose re-binned sample is the + // only point representation it will ever have. + if (view.comm) return null; const cache = g.densityCache || (g.density ? [g.density] : []); let contained = null; let fallback = null; @@ -268,6 +328,7 @@ export function lodSampleForView(view, g) { const o = d && d.overlay; if (!o || !o.n || !o.win || seen.has(o)) continue; seen.add(o); + if (!lodOverlayResolvable(view, o)) continue; if (view._viewInside(o.win)) { if (!contained || lodWindowArea(o.win) < lodWindowArea(contained.win)) contained = o; } else if (view._viewOverlaps(o.win)) { @@ -293,6 +354,176 @@ function lodDensityForView(view, g) { return best || broadest || g.density; } +// NOTE (recorded reversal, T13): a "finer-detail layer" — drawing the +// smallest cached texture overlapping the view on top of the broad backdrop +// during pans — was tried here and reverted after a field capture showed it +// as a stale darker rectangle: density textures alpha-composite, so the +// overlap region double-counts opacity, and each window's texture is baked +// against its own eased normMax, so the seam is also a brightness step. +// Doing it right needs the backdrop scissored out of the detail region plus +// a shared normalization across cached textures (or kernel-side padded +// aligned density windows, so the primary texture simply keeps containing +// the view). Until then the tier draws ONE texture per frame. + +// T13 (revised on #225 field feedback): the aggregate tier does NOT refine. +// Whatever density texture already covers the view stands — however blurry — +// until the view could plausibly resolve into REAL points; only then is a +// round-trip worth anything, and the kernel answers it with exact points +// once the window's count fits the budget. Blur at intermediate zooms is an +// accepted cost: a refreshed aggregate at 300% zoom is the same picture with +// marginally softer edges, and the field capture behind the old refinement +// loop paid megabytes per pan/zoom step for it. +// +// The estimate comes from the smallest cached density window CONTAINING the +// view — its recorded count is exact for its window, area-scaled down to the +// view (home's total count is the fallback everything starts from). The +// band factor absorbs area-scaling error against non-uniform data: dense +// cores under-estimate (a wasted-but-small transition request whose density +// reply recalibrates the local count), sparse tails over-estimate (points +// arrive a few zoom steps later than ideal — more of the accepted blur). A +// trace with no recorded counts anywhere always requests: silence must +// never be the default when the kernel might hold a sharper representation. +const LOD_POINTS_REQUEST_BAND = 4; // × LOD_DIRECT_POINT_BUDGET + +// Stepped aggregate ladder (T13): between the home texture and real points +// the aggregate sharpens in QUANTIZED steps, never per-view. A step-k +// texture covers the view snapped outward to a power-of-STEP_FACTOR block +// grid over the data extent, per axis — the same aligned-window law as the +// kernel's padded drills — so every view in a region resolves to the SAME +// window (pan-stable, dedupable, cacheable) and a zoom sees at most +// STEP_MAX smooth-to-smooth texture swaps before points land. Per-view +// windows were the field-captured failure mode (a fresh texture per +// pan/zoom step reads as jumping); the ladder bounds worst-case softness +// (≈ STEP_FACTOR× stretch per axis) instead of eliminating blur. +const LOD_AGG_STEP_FACTOR = 4; // block span shrinks 4× per step, per axis +const LOD_AGG_STEP_MAX = 2; // intermediate steps below home before points +// A covering texture within this area factor of the desired step window is +// "fine enough" — no upgrade request. +const LOD_AGG_STEP_SLACK = 1.5; + +function lodAlignedStepSpan(lo, hi, e0, e1, blocks) { + const extent = e1 - e0; + if (!(extent > 0) || !(blocks >= 1)) return [lo, hi]; + const block = extent / blocks; + const b0 = Math.floor((lo - e0) / block); + const b1 = Math.ceil((hi - e0) / block); + // min/max guards keep containment through float rounding at block edges. + return [Math.min(lo, e0 + b0 * block), Math.max(hi, e0 + b1 * block)]; +} + +// The aligned step window the view's aggregate SHOULD be served at, or null +// when no upgrade is needed (home is step 0; a covering cached texture +// already at — or finer than — the desired step satisfies it). +export function lodAggregateStepWindow(view, g, x0, x1, y0, y1) { + const cache = g.densityCache || (g.density ? [g.density] : []); + const vx0 = Math.min(x0, x1), vx1 = Math.max(x0, x1); + const vy0 = Math.min(y0, y1), vy1 = Math.max(y0, y1); + const vSpanX = vx1 - vx0, vSpanY = vy1 - vy0; + if (!(vSpanX > 0 && vSpanY > 0)) return null; + // The broadest textured entry is the extent anchor (the home texture). + let extent = null; + for (const d of cache) { + if (!d || !d.tex || !d.xRange || !d.yRange) continue; + if (!extent || lodDensityArea(d) > lodDensityArea(extent)) extent = d; + } + if (!extent) return null; + const ex0 = Math.min(extent.xRange[0], extent.xRange[1]); + const ex1 = Math.max(extent.xRange[0], extent.xRange[1]); + const ey0 = Math.min(extent.yRange[0], extent.yRange[1]); + const ey1 = Math.max(extent.yRange[0], extent.yRange[1]); + const exSpan = ex1 - ex0, eySpan = ey1 - ey0; + if (!(exSpan > 0 && eySpan > 0)) return null; + const stepOf = (extentSpan, viewSpan) => Math.max(0, Math.min( + LOD_AGG_STEP_MAX, + Math.floor(Math.log(extentSpan / Math.max(viewSpan, 1e-300)) / Math.log(LOD_AGG_STEP_FACTOR)), + )); + const kx = stepOf(exSpan, vSpanX); + const ky = stepOf(eySpan, vSpanY); + if (kx === 0 && ky === 0) return null; + const wx = lodAlignedStepSpan(vx0, vx1, ex0, ex1, Math.pow(LOD_AGG_STEP_FACTOR, kx)); + const wy = lodAlignedStepSpan(vy0, vy1, ey0, ey1, Math.pow(LOD_AGG_STEP_FACTOR, ky)); + const winArea = (wx[1] - wx[0]) * (wy[1] - wy[0]); + if (!(winArea > 0)) return null; + // Fine enough already? The smallest covering TEXTURE decides. + let covering = null; + const exEps = vSpanX * 1e-4, eyEps = vSpanY * 1e-4; + for (const d of cache) { + if (!d || !d.tex || !d.xRange || !d.yRange) continue; + const cx0 = Math.min(d.xRange[0], d.xRange[1]), cx1 = Math.max(d.xRange[0], d.xRange[1]); + const cy0 = Math.min(d.yRange[0], d.yRange[1]), cy1 = Math.max(d.yRange[0], d.yRange[1]); + if (vx0 < cx0 - exEps || vx1 > cx1 + exEps || vy0 < cy0 - eyEps || vy1 > cy1 + eyEps) continue; + if (!covering || lodDensityArea(d) < lodDensityArea(covering)) covering = d; + } + if (covering && lodDensityArea(covering) <= winArea * LOD_AGG_STEP_SLACK) return null; + return [wx[0], wx[1], wy[0], wy[1]]; +} + +// Unbiased local count estimate from the retained deterministic sample: the +// first-payload sample is a fixed-rate thinning of the whole trace, so +// (in-view sample rows) × (total ÷ sample size) estimates the in-view count +// following the data's ACTUAL distribution — with ~65 expected sample rows +// right at the gating band, ±12% noise. Area-scaling alone cannot do this: +// it assumes uniform density, over-estimating sparse tails by orders of +// magnitude, which under the stands-rule would strand them in blur far past +// the zoom where real points are available. Kernel-attached clients never +// DRAW this sample (T9); estimating from it CPU-side is exactly the kind of +// use it is retained for (like the standalone re-bin worker). NaN when no +// sample is retained (overlay omitted past u32 rows, hand-built specs). +function lodSampleViewCount(view, g, x0, x1, y0, y1) { + const o = g.sampleOverlay; + const cpu = o && o._cpu; + const meta = o && o.sample; + if (!cpu || !cpu.x || !cpu.y || !meta || !Number.isFinite(meta.visible) || !(meta.n > 0)) { + return NaN; + } + // Containment tested in the sample's own encoded space (§16): enc = + // (raw − offset) × scale, with a swap guard for negative scales. + const sx = cpu.xMeta.scale || 1, ox = cpu.xMeta.offset || 0; + const sy = cpu.yMeta.scale || 1, oy = cpu.yMeta.offset || 0; + let ex0 = (Math.min(x0, x1) - ox) * sx, ex1 = (Math.max(x0, x1) - ox) * sx; + if (ex0 > ex1) { const t = ex0; ex0 = ex1; ex1 = t; } + let ey0 = (Math.min(y0, y1) - oy) * sy, ey1 = (Math.max(y0, y1) - oy) * sy; + if (ey0 > ey1) { const t = ey0; ey0 = ey1; ey1 = t; } + const n = Math.min(cpu.x.length, cpu.y.length); + let k = 0; + for (let i = 0; i < n; i++) { + const xv = cpu.x[i], yv = cpu.y[i]; + if (xv >= ex0 && xv <= ex1 && yv >= ey0 && yv <= ey1) k++; + } + return k * (meta.visible / meta.n); +} + +export function lodAggregateStands(view, g, x0, x1, y0, y1) { + const cache = g.densityCache || (g.density ? [g.density] : []); + const vx0 = Math.min(x0, x1), vx1 = Math.max(x0, x1); + const vy0 = Math.min(y0, y1), vy1 = Math.max(y0, y1); + const vSpanX = vx1 - vx0, vSpanY = vy1 - vy0; + if (!(vSpanX > 0 && vSpanY > 0)) return false; + const ex = vSpanX * 1e-4, ey = vSpanY * 1e-4; + let winArea = 0; + let est = NaN; + for (const d of cache) { + if (!d || !d.xRange || !d.yRange || !Number.isFinite(d.visible)) continue; + const wx0 = Math.min(d.xRange[0], d.xRange[1]), wx1 = Math.max(d.xRange[0], d.xRange[1]); + const wy0 = Math.min(d.yRange[0], d.yRange[1]), wy1 = Math.max(d.yRange[0], d.yRange[1]); + if (vx0 < wx0 - ex || vx1 > wx1 + ex || vy0 < wy0 - ey || vy1 > wy1 + ey) continue; + const a = (wx1 - wx0) * (wy1 - wy0); + if (!(a > 0)) continue; + if (!(winArea > 0) || a < winArea) { + winArea = a; + est = d.visible * Math.min(1, (vSpanX * vSpanY) / a); + } + } + // Two independent estimators, keep the LOWER: never let an over-estimate + // hold a view in blur when either signal says points could be close. The + // area estimate under-shoots dense cores (a small early transition request + // whose reply recalibrates); the sample estimate is distribution-true. + const sampleEst = lodSampleViewCount(view, g, x0, x1, y0, y1); + if (Number.isFinite(sampleEst)) est = Number.isFinite(est) ? Math.min(est, sampleEst) : sampleEst; + if (!Number.isFinite(est)) return false; + return est > LOD_DIRECT_POINT_BUDGET * LOD_POINTS_REQUEST_BAND; +} + // The hold's density estimate, reused by retirement (T11): scale the drill // window's known count by the target window's area to predict whether that // window could still be answered with direct points. @@ -328,13 +559,14 @@ function lodHoldPendingDrill(view, g, d) { // different representation and the reply flow owns that transition. const LOD_DRILL_REENCODE_SPAN = 1 / 256; -export function lodDrillServesView(g, x0, x1, y0, y1) { - const d = g && g.drill; - if (!d || !d.exact || !d.win || g._drillDying) return false; +// Containment + §16 re-encode depth bound, shared by the live drill and the +// retired point-window cache (T13): an exact window answers any view it +// contains, until the zoom outgrows the f32 offset encoding. +function lodWindowServesView(win, x0, x1, y0, y1) { const vx0 = Math.min(x0, x1), vx1 = Math.max(x0, x1); const vy0 = Math.min(y0, y1), vy1 = Math.max(y0, y1); - const wx0 = Math.min(d.win.x0, d.win.x1), wx1 = Math.max(d.win.x0, d.win.x1); - const wy0 = Math.min(d.win.y0, d.win.y1), wy1 = Math.max(d.win.y0, d.win.y1); + const wx0 = Math.min(win.x0, win.x1), wx1 = Math.max(win.x0, win.x1); + const wy0 = Math.min(win.y0, win.y1), wy1 = Math.max(win.y0, win.y1); // Same edge tolerance as _viewInside: f32 round-trip slop at the window // boundary must not force a request right after drilling in. const ex = (vx1 - vx0) * 1e-4, ey = (vy1 - vy0) * 1e-4; @@ -345,6 +577,12 @@ export function lodDrillServesView(g, x0, x1, y0, y1) { ); } +export function lodDrillServesView(g, x0, x1, y0, y1) { + const d = g && g.drill; + if (!d || !d.exact || !d.win || g._drillDying) return false; + return lodWindowServesView(d.win, x0, x1, y0, y1); +} + // Geometry-only retirement (T11): an entered-then-exited drill is kept as a // revive cache — a rapid zoom back into its window hands the exact marks // back with no kernel round-trip — but only while a nearby view could still @@ -375,11 +613,35 @@ function lodDensityPinned(g, d) { d === g._shownDensity || d === g._homeDensity; } +function lodSameDensityWindow(a, b) { + if (!a || !b || !a.xRange || !b.xRange || !a.yRange || !b.yRange) return false; + const eps = + (Math.abs(a.xRange[1] - a.xRange[0]) + Math.abs(a.yRange[1] - a.yRange[0])) * 1e-9 + 1e-300; + return Math.abs(a.xRange[0] - b.xRange[0]) <= eps && Math.abs(a.xRange[1] - b.xRange[1]) <= eps && + Math.abs(a.yRange[0] - b.yRange[0]) <= eps && Math.abs(a.yRange[1] - b.yRange[1]) <= eps; +} + export function lodRememberDensity(view, g, d) { if (!d || !d.tex) return; d._stamp = ++view._densityStamp; if (!g.densityCache) g.densityCache = []; - if (!g.densityCache.includes(d)) g.densityCache.push(d); + if (!g.densityCache.includes(d)) { + // A fresh grid for a window already cached supersedes its twin: same + // coverage, newer facts (count, resolution, min_cell) — and request + // elision reads those facts, so a stale twin must not shadow them. + // Pinned twins (mid-crossfade, home) stay; eviction sweeps them later. + for (let i = g.densityCache.length - 1; i >= 0; i--) { + const old = g.densityCache[i]; + if (old === d || lodDensityPinned(g, old) || !lodSameDensityWindow(old, d)) continue; + g.densityCache.splice(i, 1); + if (old.tex) view.gl.deleteTexture(old.tex); + if (old.overlay && old.overlay !== g.sampleOverlay) { + view._destroySampleOverlay(old.overlay); + old.overlay = null; + } + } + g.densityCache.push(d); + } const maxCached = 8; while (g.densityCache.length > maxCached) { let drop = -1; @@ -396,7 +658,7 @@ export function lodRememberDensity(view, g, d) { if (drop < 0) break; const old = g.densityCache.splice(drop, 1)[0]; if (!lodDensityPinned(g, old)) { - view.gl.deleteTexture(old.tex); + if (old.tex) view.gl.deleteTexture(old.tex); // The window's sample overlay rides its cache entry (T9 pairing) and // dies with it — except the home/init overlay, which the standalone // re-bin worker keeps as its CPU-side source. @@ -408,6 +670,116 @@ export function lodRememberDensity(view, g, d) { } } +// -- point-window cache (T13) ------------------------------------------------- +// +// Retired exact drills, LRU per trace. "Once we get points, we can render +// anything inside there without further requests": a view covered by any +// cached full-point window promotes it back to the live drill with no wire +// round-trip. The cache pairs with the kernel's padded ALIGNED windows — +// consecutive pans resolve to the same aligned bounds, so windows crossed +// once are held, and ping-pong pans/zooms across a boundary re-render from +// the cache instead of re-shipping ~all the same points. + +function lodSameWindow(a, b) { + if (!a || !b) return false; + const eps = (Math.abs(a.x1 - a.x0) + Math.abs(a.y1 - a.y0)) * 1e-9 + 1e-300; + return Math.abs(a.x0 - b.x0) <= eps && Math.abs(a.x1 - b.x1) <= eps && + Math.abs(a.y0 - b.y0) <= eps && Math.abs(a.y1 - b.y1) <= eps; +} + +function lodFreeDrillBuffers(view, d) { + const gl = view.gl; + view._deleteVaos(d); // each drill object carries its own VAOs + for (const b of [d.xBuf, d.yBuf, d.cBuf, d.rgbaBuf, d.sBuf, d.styleBuf, + d.strokeBuf, d.selBuf, d.dBuf]) if (b) gl.deleteBuffer(b); +} + +// Reset the trace's drill lifecycle state without touching the (moved or +// freed) drill object itself — shared by drop, retire, and promote. +function lodClearDrillState(view, g) { + g.drill = null; + g._drillFadeStart = null; + g._drillExitFadeStart = null; + g._drillWasInside = false; + g._drillEverInside = false; + g._drillShownAlpha = null; + g._drillDying = false; + g._drillDiedInsideWin = false; + g._drillBackdropShown = 1; // next drill enters over a full backdrop (T10) + g._drillBackdropTick = 0; + view._hoverId = -1; // drilled indices are dead; don't reuse a cached row + view._lastRow = null; + // The freed drill may have been the only pickable geometry (a density-only + // chart): retract the modebar Select trigger with the capability. + view._updatePickable(); +} + +// Move the live drill into the retired-window cache. Only an exact subset can +// serve future views (Invariant L2), so anything else frees instead. +function lodRetireDrill(view, g) { + const d = g.drill; + if (!d) return; + if (!d.exact || !d.win) { + lodDropDrill(view, g); + return; + } + if (!g.drillCache) g.drillCache = []; + // A re-shipped window replaces its stale cached twin instead of duplicating. + for (let i = g.drillCache.length - 1; i >= 0; i--) { + if (lodSameWindow(g.drillCache[i].win, d.win)) { + lodFreeDrillBuffers(view, g.drillCache.splice(i, 1)[0]); + } + } + g.drillCache.push(d); + while (g.drillCache.length > LOD_POINT_CACHE_WINDOWS) { + lodFreeDrillBuffers(view, g.drillCache.shift()); + } + lodClearDrillState(view, g); +} + +export function lodDropPointCache(view, g) { + for (const d of g.drillCache || []) lodFreeDrillBuffers(view, d); + g.drillCache = null; +} + +// A retired cached window that covers the view swaps back in as the live +// drill — the pan-back / zoom-ping-pong "render anything inside there" path +// (T13). Alpha-continuous like a revive: the swap hands marks over at the +// alpha currently on screen, and any active brush mask re-derives locally +// exactly as a fresh reply would (§34 continuity). +export function lodPromoteCachedDrill(view, g, x0, x1, y0, y1) { + const cache = g.drillCache; + if (!cache || !cache.length) return false; + let pick = -1; + for (let i = 0; i < cache.length; i++) { + const e = cache[i]; + if (!e || !e.exact || !e.win || !lodWindowServesView(e.win, x0, x1, y0, y1)) continue; + // Smallest covering window wins (mirrors lodDensityForView): tightest + // offset encoding and the most local blend/density metadata. + if (pick < 0 || lodWindowArea(e.win) < lodWindowArea(cache[pick].win)) pick = i; + } + if (pick < 0) return false; + const e = cache.splice(pick, 1)[0]; + const shownAlpha = g.drill ? lodDrillShownAlpha(view, g) : 0; + if (g.drill) { + if (g._drillDying || !g.drill.exact) lodDropDrill(view, g); + else lodRetireDrill(view, g); + } else { + lodClearDrillState(view, g); + } + g.drill = e; + // Continuity across the swap: the promoted window's marks pick up at the + // alpha the retired ones showed (the retired window's points are a subset/ + // superset over the same region — a restart-from-zero reads as a blink). + g._drillShownAlpha = shownAlpha; + lodEnterDrillContinuous(view, g); + if (view._lastBrush && e._cpuX && e._cpuY) lodRestoreBrushMask(view, e, e._cpuX, e._cpuY); + else e.selActive = false; + view._updatePickable(); + view.draw(); + return true; +} + // -- drill lifecycle ---------------------------------------------------------- // The kernel decided this view fits the direct budget and shipped real marks @@ -415,7 +787,27 @@ export function lodRememberDensity(view, g, d) { // trace; the tier draw uses it until the kernel switches back. export function lodApplyDrill(view, g, upd, buffers) { const gl = view.gl; - const fresh = !g.drill; // transition INTO drill vs refresh of a live drill + g._stepReqWin = null; // real points supersede any outstanding ladder step + const win = { + x0: upd.x_range[0], x1: upd.x_range[1], + y0: upd.y_range[0], y1: upd.y_range[1], + }; + // A reply for a NEW window retires the old one into the point-window cache + // (T13) instead of overwriting its buffers: the old window's points remain + // exact for views inside it, so a pan back promotes them with no kernel + // round-trip. The handoff is alpha-continuous below — a window-to-window + // swap must not restart the entry fade (reads as flashing mid-pan). + let handoff = null; + if (g.drill && !lodSameWindow(g.drill.win, win)) { + handoff = { + shownAlpha: lodDrillShownAlpha(view, g), + wasInside: g._drillWasInside, + everInside: g._drillEverInside, + }; + if (g._drillDying || !g.drill.exact) lodDropDrill(view, g); + else lodRetireDrill(view, g); + } + const fresh = !g.drill && !handoff; // true aggregate→marks transition let d = g.drill; if (!d) { d = g.drill = { trace: g.trace, xBuf: gl.createBuffer(), yBuf: gl.createBuffer() }; @@ -431,9 +823,14 @@ export function lodApplyDrill(view, g, upd, buffers) { gl.bufferData(gl.ARRAY_BUFFER, ys, gl.STATIC_DRAW); d.xMeta = { offset: upd.x.offset, scale: upd.x.scale }; d.yMeta = { offset: upd.y.offset, scale: upd.y.scale }; - d.win = { x0: upd.x_range[0], x1: upd.x_range[1], y0: upd.y_range[0], y1: upd.y_range[1] }; + d.win = win; d.n = Math.min(upd.x.len, upd.y.len); d.visible = upd.visible ?? d.n; + // Encoded coordinates retained CPU-side (views over the reply frame, no + // copy): a promoted cached window re-derives the brush mask from these, + // exactly as this fresh reply does below (§34 continuity across T13 swaps). + d._cpuX = xs; + d._cpuY = ys; // The kernel's exactness claim (§28 Invariant L2): reduction "none" means // the subset IS every point in the window — the fact that arms T12's // zoom-in request elision. Anything else (or a reply that doesn't say) @@ -521,12 +918,15 @@ export function lodApplyDrill(view, g, upd, buffers) { gl.bufferData(gl.ARRAY_BUFFER, values, gl.STATIC_DRAW); } view._pointMarkStyle(d, d.trace); - // Intensity-continuous handoff (§5): per-point local log-density + a blend - // weight. The density surface already wears the mean point color (LOD doc - // §2), so hue is continuous by construction; fresh at the boundary - // (blend≈1) each mark enters at its cell's count-alpha and deeper zooms - // ship smaller blends, easing marks to native opacity. - if (upd.density_val && upd.density_val.buf !== undefined) { + // Intensity handoff (§5): per-point local log-density + a blend weight. + // A mean-color surface needs NONE of it (§2, physical alpha): its texture + // composites exactly like the points themselves, so marks enter and exit + // at native opacity and the entry/exit alpha fades alone carry the swap — + // running the count-alpha ramp against a physically-composited backdrop + // would re-introduce the lightness step it exists to avoid. Count-only + // surfaces keep the ramp (their texture still wears the log tone curve). + const meanColorSurface = !!(g.density && g.density.rgba); + if (!meanColorSurface && upd.density_val && upd.density_val.buf !== undefined) { const dvalValues = upd.density_val.dtype === "u8" ? view._asU8(buffers[upd.density_val.buf]) : view._asF32(buffers[upd.density_val.buf]); @@ -562,6 +962,18 @@ export function lodApplyDrill(view, g, upd, buffers) { g._drillDiedInsideWin = false; return; } + // Window-to-window swap (the old window retired into the point cache): + // marks continue from the alpha the retired window showed — the two windows + // agree on every shared point, so a restart-from-zero reads as a blink. + if (handoff) { + g._drillWasInside = handoff.wasInside; + g._drillEverInside = handoff.everInside; + g._drillShownAlpha = handoff.shownAlpha; + g._drillDying = false; + g._drillDiedInsideWin = false; + lodEnterDrillContinuous(view, g); + return; + } // A live points reply revives a dying/exiting drill (hysteresis flip or a // fast zoom back in): hand the marks back at their CURRENT alpha — neither // fighting the exit fade nor snapping to full. @@ -607,25 +1019,8 @@ function lodRestoreBrushMask(view, d, xs, ys) { export function lodDropDrill(view, g) { const d = g.drill; if (!d) return; - const gl = view.gl; - view._deleteVaos(d); // the drill sibling carries its own VAOs - for (const b of [d.xBuf, d.yBuf, d.cBuf, d.rgbaBuf, d.sBuf, d.styleBuf, - d.strokeBuf, d.selBuf, d.dBuf]) if (b) gl.deleteBuffer(b); - g.drill = null; - g._drillFadeStart = null; - g._drillExitFadeStart = null; - g._drillWasInside = false; - g._drillEverInside = false; - g._drillShownAlpha = null; - g._drillDying = false; - g._drillDiedInsideWin = false; - g._drillBackdropShown = 1; // next drill enters over a full backdrop (T10) - g._drillBackdropTick = 0; - view._hoverId = -1; // drilled indices are dead; don't reuse a cached row - view._lastRow = null; - // The freed drill may have been the only pickable geometry (a density-only - // chart): retract the modebar Select trigger with the capability. - view._updatePickable(); + lodFreeDrillBuffers(view, d); + lodClearDrillState(view, g); } // A density update arrived while drilled: don't drop the marks instantly @@ -717,12 +1112,72 @@ function lodBeginDrillExitContinuous(view, g) { // -- aggregate updates & tier drawing ---------------------------------------- +// Facts-only cache entry for a display-suppressed aggregate reply (T13): no +// texture, just the window and its exact count — the points-band gate +// (lodAggregateStands) reads it to recalibrate local estimates while the +// standing texture keeps owning the frame. +function lodRememberDensityFacts(view, g, upd) { + const d = upd.density; + if (!d || !Array.isArray(d.x_range) || !Array.isArray(d.y_range)) return; + if (!Number.isFinite(upd.visible)) return; + const entry: any = { + xRange: [d.x_range[0], d.x_range[1]], + yRange: [d.y_range[0], d.y_range[1]], + visible: upd.visible, + }; + if (!g.densityCache) g.densityCache = []; + for (let i = g.densityCache.length - 1; i >= 0; i--) { + const old = g.densityCache[i]; + if (lodDensityPinned(g, old) || !lodSameDensityWindow(old, entry)) continue; + if (old.tex) continue; // never displace a real texture with bare facts + g.densityCache.splice(i, 1); + } + entry._stamp = ++view._densityStamp; + g.densityCache.push(entry); + while (g.densityCache.length > 12) { + const idx = g.densityCache.findIndex((c) => c && !c.tex && c !== entry); + if (idx < 0) break; + g.densityCache.splice(idx, 1); + } +} + // Apply a kernel "density"-mode update: new grid texture with eased exposure // normalization, previous grid kept for the crossfade, source remembered in // the per-trace cache. export function lodApplyDensityUpdate(view, g, upd, buffers) { lodMarkDrillDying(view, g); const d = upd.density; + // Display side of the stands-rule (T13, user-directed #225 follow-up): + // with a kernel attached, an aggregate reply REPLACES a covering texture + // only when the client asked for exactly this window as a LADDER STEP + // (the quantized aligned windows of the stepped aggregate — smooth, + // pan-stable, at most STEP_MAX swaps before points). Every other covered + // reply — the points-band probes above all — lands as FACTS only (its + // window's exact count recalibrates the gate): the band's exact grids + // have a hard speckled character (sparse cells at the points' own + // alpha), and repainting the smooth surface with one on every probe read + // as jumping between zoom levels in the field capture. The drill + // lifecycle above proceeds either way — dying marks fade into the + // standing surface. Coverage failure (a pan past every cached window) + // still applies the reply: silence must never blank a frame (T1). + // Standalone clients keep applying everything — their re-binned grids + // are the only refinement they have. + if (view.comm) { + const sw = g._stepReqWin; + const tol = sw ? (Math.abs(sw[1] - sw[0]) + Math.abs(sw[3] - sw[2])) * 1e-6 + 1e-300 : 0; + const isStepReply = !!sw && Array.isArray(d.x_range) && Array.isArray(d.y_range) && + Math.abs(d.x_range[0] - sw[0]) <= tol && Math.abs(d.x_range[1] - sw[1]) <= tol && + Math.abs(d.y_range[0] - sw[2]) <= tol && Math.abs(d.y_range[1] - sw[3]) <= tol; + if (isStepReply) { + g._stepReqWin = null; + } else { + const covering = lodDensityForView(view, g); + if (covering && covering.tex && view._viewInsideRange(covering.xRange, covering.yRange)) { + lodRememberDensityFacts(view, g, upd); + return; + } + } + } const grid = d.enc === "log-u8" ? lodDecodeLogU8(buffers[d.buf], d.max) : lodCopyGrid(view._asF32(buffers[d.buf])); @@ -739,10 +1194,16 @@ export function lodApplyDensityUpdate(view, g, upd, buffers) { w: d.w, h: d.h, max: d.max, normMax, colormap: d.colormap || g.density.colormap, color: d.color ? parseColor(view.root, d.color, [0.3, 0.47, 0.66, 1]) : g.density.color, xRange: d.x_range, yRange: d.y_range, + // Request-gating fact (T13): the window's exact point count — the local + // estimate lodAggregateStands scales to decide whether a view is close + // enough to points territory to be worth a round-trip at all. + visible: upd.visible, grid, rgba, filter, - tex: view._uploadGrid(grid, d.w, d.h, normMax, rgba, filter), + tex: view._uploadGrid( + grid, d.w, d.h, normMax, rgba, filter, view._fillOpacity(g.trace.style), + ), lut: g.density.lut, }; // Exact scans include a view-specific sample and replace the overlay. @@ -753,7 +1214,11 @@ export function lodApplyDensityUpdate(view, g, upd, buffers) { if (Object.prototype.hasOwnProperty.call(d, "sample")) { view._applyDensitySample(g, d.sample, buffers); } - lodStartNormAnim(view, g, normMax, d.max); + // Exposure easing (T4) is a count-only concern: a mean-color texture's + // physical alpha is max-independent, so re-uploading it per norm step + // would rewrite identical bytes. + if (rgba) g._densityNormAnim = null; + else lodStartNormAnim(view, g, normMax, d.max); lodRememberDensity(view, g, g.density); } @@ -824,6 +1289,18 @@ function lodDrillBackdropScale(view, g, target) { // goes pending. export function lodDrawDensityTier(view, g, x0, x1, y0, y1) { lodStepNorm(view, g); + // Retired point windows obey the same geometry-only discipline as the live + // drill (T11, via T13): once the view outgrows a cached window past the + // drill budget, no nearby view could be served by it, so its GPU buffers + // free on this frame — §27's rebuildable-cache rule, no kernel reply needed. + if (g.drillCache) { + for (let i = g.drillCache.length - 1; i >= 0; i--) { + if (lodDrillOutgrown(view, g, g.drillCache[i])) { + lodFreeDrillBuffers(view, g.drillCache.splice(i, 1)[0]); + } + } + if (!g.drillCache.length) g.drillCache = null; + } const d = g.drill; // Rapid zoom out→in revive: a dying drill whose window still covers the // view is exact for it (the subset IS every point in that window). Cancel @@ -906,13 +1383,23 @@ export function lodDrawDensityTier(view, g, x0, x1, y0, y1) { drawMarks(1 - exitFade); view.draw(); } else { - if (g._drillDying) lodDropDrill(view, g); // fade done: free the buffers - else if (exitingDrill) g._drillWasInside = false; + if (g._drillDying) { + // Fade done. A subset that died OUTSIDE its window is still exact for + // it — the kernel chose density for a view the window doesn't cover — + // so it retires into the point cache (T13) and a zoom back in + // promotes it with no round-trip. Dying INSIDE means the kernel chose + // density FOR this window's own views (forced density / data change): + // its points-tier claim is void, free the buffers. + if (d.exact && !g._drillDiedInsideWin) lodRetireDrill(view, g); + else lodDropDrill(view, g); + } else if (exitingDrill) { + g._drillWasInside = false; + } // Geometry-only retirement (T11): an entered drill whose exit has // completed frees its buffers once the view outgrows its window past // the drill budget — no kernel reply required, and it must run on the // completion frame itself (a settled view schedules no further frames). - // A dying drill was just dropped above; a never-entered prefetch is + // A dying drill was just handled above; a never-entered prefetch is // exempt (its window is simply ahead of the view). if (g.drill && g._drillEverInside && lodDrillOutgrown(view, g, d)) { lodDropDrill(view, g); diff --git a/js/src/50_chartview.ts b/js/src/50_chartview.ts index eeac5ca3..c9fb3fc2 100644 --- a/js/src/50_chartview.ts +++ b/js/src/50_chartview.ts @@ -3,7 +3,7 @@ import { buildLutData, colormapStops } from "./10_colormaps"; import { cssColor, ensureChromeStylesheet, hexColor, parseColor, readTheme, safeCssPaint } from "./20_theme"; import { categoryTicks, fmtAxis, fmtGeneral, fmtLinear, fmtValue, linearTicks, logTicks, timeTicks } from "./30_ticks"; import { AREA_FS, AREA_VS, ATTR_SLOTS, BAR_VS, DENSITY_FS, GRID_VS, HEATMAP_FS, LINE_FS, LINE_VS, MESH_FS, MESH_VS, PICK_FS, PICK_VS, POINT_FS, POINT_SIMPLE_FS, POINT_SIMPLE_VS, POINT_VS, RECT_FS, RECT_VS, SEGMENT_FS, SEGMENT_VS, makeProgram, uniformOf, xySmoothResample } from "./40_gl"; -import { lodCopyGrid, lodDecodeLogU8, lodDrawDensityTier, lodRememberDensity, lodSampleForView, lodWriteGridTexture } from "./45_lod"; +import { lodCopyGrid, lodDecodeLogU8, lodDrawDensityTier, lodDropPointCache, lodRememberDensity, lodSampleForView, lodWriteGridTexture } from "./45_lod"; import { markOf } from "./55_marks"; // --------------------------------------------------------------------------- @@ -2110,10 +2110,14 @@ export class ChartView { w: d.w, h: d.h, max: d.max, normMax: d.max, colormap: d.colormap, color: d.color ? parseColor(this.root, d.color, [0.3, 0.47, 0.66, 1]) : null, xRange: d.x_range, yRange: d.y_range, + // The home window's count seeds lodAggregateStands (T13): every + // zoom's points-band estimate starts from this until a closer + // window's reply recalibrates it. + visible: t.visible, grid: lodCopyGrid(grid), rgba, filter, - tex: this._uploadGrid(grid, d.w, d.h, d.max, rgba, filter), + tex: this._uploadGrid(grid, d.w, d.h, d.max, rgba, filter, this._fillOpacity(t.style)), lut: this._lut(d.colormap), }; g.sampleOverlay = this._buildDensitySample(t, d.sample, buffer); @@ -2860,10 +2864,10 @@ export class ChartView { return tex; } - _uploadGrid(f32, w, h, maxVal, rgba = null, filter = "linear") { + _uploadGrid(f32, w, h, maxVal, rgba = null, filter = "linear", pointAlpha = 1) { const gl = this.gl; const tex = gl.createTexture(); - lodWriteGridTexture(gl, tex, f32, w, h, maxVal, rgba, filter); + lodWriteGridTexture(gl, tex, f32, w, h, maxVal, rgba, filter, pointAlpha); return tex; } @@ -3449,7 +3453,14 @@ export class ChartView { this._axisCoord(xAxis, d.xRange[0]), this._axisCoord(xAxis, d.xRange[1]), this._axisCoord(yAxis, d.yRange[0]), this._axisCoord(yAxis, d.yRange[1]), ); - gl.uniform1f(u("u_opacity"), this._fillOpacity(g.trace.style) * opacityScale); + // Mean-color textures bake the style opacity INSIDE their physical + // compositing (LOD doc §2 rule 1 — dense cells saturate past it exactly + // like overplotted marks), so the uniform carries only the transition + // fades for them; count-only grids keep style opacity here as before. + gl.uniform1f( + u("u_opacity"), + (d.rgba ? 1 : this._fillOpacity(g.trace.style)) * opacityScale, + ); // Mean-color grids carry their colors in the texture (LOD doc §2); // count-only grids tint with the constant trace color or, failing that, // fall back to the LUT ramp (hand-built/legacy specs). @@ -5085,6 +5096,7 @@ export class ChartView { _destroyTraceResources(g, texSeen) { if (!g) return; this._destroyDensitySample(g); + lodDropPointCache(this, g); // retired point windows die with the trace (T13) this._deleteVaos(g); this._deleteVaos(g.drill); this._deleteBuffers(g, [ @@ -5094,7 +5106,9 @@ export class ChartView { "_transitionPrevXBuf", "_transitionPrevYBuf", "_transitionPrevPosBuf", "_transitionPrevValue1Buf", "_transitionPrevValue0Buf", ]); - this._deleteBuffers(g.drill, ["xBuf", "yBuf", "cBuf", "sBuf", "selBuf", "dBuf"]); + this._deleteBuffers(g.drill, [ + "xBuf", "yBuf", "cBuf", "rgbaBuf", "sBuf", "styleBuf", "strokeBuf", "selBuf", "dBuf", + ]); const textures = []; if (g.heatmap) textures.push(g.heatmap.tex); for (const d of g.densityCache || []) textures.push(d && d.tex); diff --git a/js/src/54_kernel.ts b/js/src/54_kernel.ts index 9511a6f6..b3859bbe 100644 --- a/js/src/54_kernel.ts +++ b/js/src/54_kernel.ts @@ -2,7 +2,8 @@ import { payloadBuffers } from "./00_header"; import { buildLutData } from "./10_colormaps"; import { parseColor } from "./20_theme"; import { - lodApplyDensityUpdate, lodApplyDrill, lodDrillServesView, lodDropDrill, lodRememberDensity, + lodAggregateStands, lodAggregateStepWindow, lodApplyDensityUpdate, lodApplyDrill, + lodDrillServesView, lodDropDrill, lodPromoteCachedDrill, lodRememberDensity, } from "./45_lod"; import { xyCreateRebinWorker } from "./46_worker"; import { ChartView } from "./50_chartview"; @@ -32,18 +33,30 @@ Object.assign(ChartView.prototype, { const now = this._now(); for (const g of this.gpuTraces) { if (g.tier !== "density") continue; - // Zoom-in request elision (T12): a view contained in an exact drill's - // window is already answered by the marks on the GPU — the smaller - // window's points are a subset of the shipped ones — so this trace + // Zoom-in request elision (T12/T13): a view contained in an exact + // window already on the GPU — the live drill, or a retired cached + // point window promoted back — is answered locally, so this trace // goes neither pending nor on the wire. The seq bump above stands, so // an in-flight reply for an older, wider view dies stale instead of // yanking the exact marks out from under the view it can't improve. - if (this._drillServesView(g, view)) { + const plan = this._drillServesView(g, view) ? null : this._densityRequestPlan(g, view); + if (!plan) { g._lodPendingView = null; g._lodPendingSeq = null; g._lodPendingAt = null; continue; } + // Duplicate of the request already in flight (identical window and + // screen): keep waiting on ITS seq instead of arming a new one that + // would kill the incoming reply just to resend the same window — + // the "constantly re-requesting the same points" loop (#225 notes). + const dup = this._densityRequestDup(g, plan.win, plotW, plotH, now); + if (dup) { + g._lodPendingView = view; + g._lodPendingSeq = dup.seq; + g._lodPendingAt = dup.sentAt; + continue; + } g._lodPendingView = view; g._lodPendingSeq = seq; g._lodPendingAt = now; @@ -72,12 +85,16 @@ Object.assign(ChartView.prototype, { }); } if (needsDensity) { + const sendNow = this._now(); for (const g of this.gpuTraces) { if (g.tier !== "density") continue; - // T12 re-check at actual send time: a drill that landed during the - // debounce elides the request it made unnecessary; one that died - // during it re-arms the request the schedule-time check skipped. - if (this._drillServesView(g, view)) { + // T12/T13 re-check at actual send time: a drill that landed during + // the debounce — or a reply that moved the view back out of the + // points band with its step already covered — elides the request + // it made unnecessary; a drill that died during it re-arms the + // request the schedule-time check skipped. + const plan = this._drillServesView(g, view) ? null : this._densityRequestPlan(g, view); + if (!plan) { if (g._lodPendingSeq === seq) { g._lodPendingView = null; g._lodPendingSeq = null; @@ -85,14 +102,30 @@ Object.assign(ChartView.prototype, { } continue; } - const [x0, x1] = this._axisRange(g.xAxis, view); - const [y0, y1] = this._axisRange(g.yAxis, view); + // Identical-request suppression (T13): the same window at the same + // screen size is either already answered (nothing to refresh — the + // reply is deterministic for unchanged data; a data change rebuilds + // this GPU record and clears the memo) or still in flight (its + // reply was adopted as this trace's pending marker above). + const dup = this._densityRequestDup(g, plan.win, plotW, plotH, sendNow); + if (dup) { + if (dup.answered && g._lodPendingSeq === seq) { + g._lodPendingView = null; + g._lodPendingSeq = null; + g._lodPendingAt = null; + } + continue; + } + const win = plan.win; this.comm.send({ type: "density_view", seq, trace: g.trace.id, - x0: Math.min(x0, x1), x1: Math.max(x0, x1), - y0: Math.min(y0, y1), y1: Math.max(y0, y1), + x0: win[0], x1: win[1], y0: win[2], y1: win[3], w: plotW, h: plotH, }); + // A ladder-step request's reply is the one density reply allowed + // to repaint a covered view (lodApplyDensityUpdate). + if (plan.step) g._stepReqWin = win; + g._lastDensityReq = { win, w: plotW, h: plotH, seq, sentAt: sendNow, answered: false }; } } }; @@ -104,6 +137,70 @@ Object.assign(ChartView.prototype, { return seq; }, + // Same request within half an output texel per edge: gesture-end and + // settle produce windows differing by sub-pixel amounts (field HAR: 0.03% + // shifts back-to-back), and a grid shifted below half a texel is visually + // identical — re-shipping it is pure wire waste. + _densityRequestSame(last, win, plotW, plotH) { + if (!last || last.w !== plotW || last.h !== plotH) return false; + const tx = (win[1] - win[0]) / plotW / 2; + const ty = (win[3] - win[2]) / plotH / 2; + return Math.abs(last.win[0] - win[0]) <= tx && Math.abs(last.win[1] - win[1]) <= tx && + Math.abs(last.win[2] - win[2]) <= ty && Math.abs(last.win[3] - win[3]) <= ty; + }, + + // The last density_view actually sent for this GPU record, when a new + // request would be its (sub-texel) twin: either already answered (the + // reply is deterministic for unchanged data, so there is nothing to + // refresh) or still in flight (bounded by the same 1200ms window as the T8 + // pending hold, so a lost reply can never suppress refresh forever). The + // memo lives on the GPU record — a rebuild (append, payload update, context + // restore) starts it fresh, so data changes are never suppressed. + _densityRequestDup(g, win, plotW, plotH, now) { + const last = g._lastDensityReq; + if (!last || !this._densityRequestSame(last, win, plotW, plotH)) return null; + if (last.answered) return last; + return now - last.sentAt < 1200 ? last : null; + }, + + // What (if anything) this trace should ask the kernel for at `view` + // (T13, revised): inside the points band, the RAW VIEW window — the + // kernel decides the tier there, and its density replies land as facts + // only; while the aggregate stands, the next LADDER STEP window when + // every covering texture is coarser than the view's step (a quantized + // aligned window, so pans resolve to the same request), else nothing. + // Drill/point-cache service is the caller's earlier check. + _densityRequestPlan(g, view) { + const [x0, x1] = this._axisRange(g.xAxis, view); + const [y0, y1] = this._axisRange(g.yAxis, view); + if (!lodAggregateStands(this, g, x0, x1, y0, y1)) { + return { + win: [Math.min(x0, x1), Math.max(x0, x1), Math.min(y0, y1), Math.max(y0, y1)], + step: false, + }; + } + const win = lodAggregateStepWindow(this, g, x0, x1, y0, y1); + return win ? { win, step: true } : null; + }, + + // A reply whose seq lost the global race can still be current in substance: + // duplicate-suppressed requests leave their traces waiting on the ORIGINAL + // request's seq (T13 — identical window and screen). Accept it only when + // every trace it updates is still waiting on exactly this seq AND that seq + // names the trace's last actually-SENT request — a pending marker can + // outlive a debounce-cancelled send, and a reply matching such a phantom + // request is stale, not suppressed-current (T5). + _densityReplyCurrent(msg) { + const ids = (msg.traces || []).map((u) => Number(u.id)); + if (!ids.length && msg.trace !== undefined) ids.push(Number(msg.trace)); + if (!ids.length) return false; + return ids.every((id) => { + const g = this.gpuTraces.find((t) => t.trace.id === id && t.tier === "density"); + return g && g._lodPendingSeq === msg.seq && + g._lastDensityReq && g._lastDensityReq.seq === msg.seq; + }); + }, + // Standalone (kernel-less) density refinement. Debounced like the kernel // request path, then the retained §28 sample re-bins in the bundled worker — // off the main thread — and applies like a density_update. @@ -150,7 +247,10 @@ Object.assign(ChartView.prototype, { const hd = g._homeDensity; this._applySampleRebinGrid(g, { ...hd, - tex: this._uploadGrid(hd.grid, hd.w, hd.h, hd.normMax || hd.max || 1, hd.rgba, hd.filter), + tex: this._uploadGrid( + hd.grid, hd.w, hd.h, hd.normMax || hd.max || 1, hd.rgba, hd.filter, + this._fillOpacity(g.trace.style), + ), }, false); } return; @@ -247,7 +347,10 @@ Object.assign(ChartView.prototype, { xRange: [msg.x0, msg.x1], yRange: [msg.y0, msg.y1], grid, rgba, - tex: this._uploadGrid(grid, msg.w, msg.h, msg.max || 1, rgba), + tex: this._uploadGrid( + grid, msg.w, msg.h, msg.max || 1, rgba, "linear", + this._fillOpacity(g.trace.style), + ), lut: g.density.lut, }, true); }, @@ -385,7 +488,7 @@ Object.assign(ChartView.prototype, { } this.draw(); } else if (msg.type === "density_update") { - if (msg.seq !== undefined && msg.seq !== this.seq) return; + if (msg.seq !== undefined && msg.seq !== this.seq && !this._densityReplyCurrent(msg)) return; const densityTraces = msg.traces || []; const pendingTraceIds = new Set(densityTraces.map((upd) => Number(upd.id))); if (pendingTraceIds.size === 0 && msg.trace !== undefined) { @@ -394,6 +497,11 @@ Object.assign(ChartView.prototype, { const clearAllPending = pendingTraceIds.size === 0 && msg.stale; const clearPending = (g) => { if (msg.seq !== undefined && g._lodPendingSeq !== msg.seq) return; + // The identical-request memo (T13): this window is now answered, so a + // later request for the same window and screen sends nothing. + if (g._lastDensityReq && g._lastDensityReq.seq === msg.seq) { + g._lastDensityReq.answered = true; + } g._lodPendingView = null; g._lodPendingSeq = null; g._lodPendingAt = null; @@ -538,15 +646,18 @@ Object.assign(ChartView.prototype, { lodDropDrill(this, g); }, - // Can `view` be answered locally from this trace's live drill, with no - // kernel round-trip (T12)? True only for an exact (reduction "none"), - // non-dying drill whose window contains the view's per-axis ranges, and - // only until the zoom outgrows the §16 f32 encode precision. + // Can `view` be answered locally with no kernel round-trip? True for an + // exact (reduction "none"), non-dying live drill whose window contains the + // view's per-axis ranges (T12), and — failing that — for any retired cached + // point window that does, which is promoted back to the live drill on the + // spot (T13). Both only until the zoom outgrows the §16 f32 encode + // precision. _drillServesView(g, view) { - if (!g.drill) return false; + if (!g) return false; const [x0, x1] = this._axisRange(g.xAxis, view); const [y0, y1] = this._axisRange(g.yAxis, view); - return lodDrillServesView(g, x0, x1, y0, y1); + if (lodDrillServesView(g, x0, x1, y0, y1)) return true; + return lodPromoteCachedDrill(this, g, x0, x1, y0, y1); }, // Is the current view fully covered by a drilled window? A tiny epsilon diff --git a/python/xy/_raster.py b/python/xy/_raster.py index ce325ec8..2514840a 100644 --- a/python/xy/_raster.py +++ b/python/xy/_raster.py @@ -41,6 +41,7 @@ _heatmap_rgba_grid, _legend_layout, _lut, + _physical_density_alpha, _px_size, _resolve_static_css_vars, _Scale, @@ -1895,24 +1896,17 @@ def _emit_grid( dx, dy, dw, dh = _scene.grid_dest_rect(xr, yr, sx, sy) if g.get("rgba") is not None: # Mean point color per cell (LOD doc §2): rgb from the shipped - # plane; count drives only the alpha ramp, scaled by the cell's - # mean point alpha — the same law as _svg._density_image and the - # client's DENSITY_FS. Precomposed here and emitted as a plain - # image op; the count→LUT density op cannot express per-cell color. + # plane; displayed alpha is the PHYSICAL compositing of the + # cell's points (`_svg._physical_density_alpha` — the same law + # as _svg._density_image and the client's texture upload). + # Precomposed here and emitted as a plain image op; the + # count→LUT density op cannot express per-cell color. rgba_meta = cols[g["rgba"]] mean = np.frombuffer( blob, dtype=np.uint8, count=rgba_meta["len"], offset=rgba_meta["byte_offset"] ).reshape(h, w, 4) counts = _density_column(blob, meta, g).reshape(h, w) - gmax = float(g.get("max") or 1.0) or 1.0 - tnorm = np.clip(counts / gmax, 0.0, 1.0) - alpha = ( - np.clip(tnorm * 1.35, 0, 1) - * 255 - * _fill_opacity(style, 0.85) - * (mean[..., 3].astype(np.float64) / 255.0) - ).astype(np.uint8) - alpha[tnorm <= 0] = 0 + alpha = _physical_density_alpha(counts, mean[..., 3], _fill_opacity(style, 0.85)) rgba = np.ascontiguousarray(np.dstack([mean[..., :3], alpha])[::-1]) cmd.image(dx, dy, dw, dh, w, h, rgba.tobytes(), nearest=False) return diff --git a/python/xy/_svg.py b/python/xy/_svg.py index f26707e3..3f235e58 100644 --- a/python/xy/_svg.py +++ b/python/xy/_svg.py @@ -2570,6 +2570,30 @@ def _grid_image( ) +def _physical_density_alpha(counts: Any, mean_alpha_u8: Any, style_opacity: float) -> Any: + """Displayed alpha of a mean-color density cell (LOD doc §2 rule 1). + + The physical compositing of the cell's own points — ``1 − (1 − a_pt)^k`` + for k points whose drawn per-point alpha is ``a_pt = channel alpha × + style opacity`` — so the surface and real marks agree on lightness at + every zoom. Style opacity folds INSIDE the exponent: dense cells + saturate past it exactly like overplotted marks do. The same law as the + client's texture upload (``lodWriteGridTexture``); shared by the SVG and + native-raster exporters. Returns u8; empty or all-invisible cells are 0. + """ + counts = np.asarray(counts, dtype=np.float64) + a8 = np.asarray(mean_alpha_u8) + a_pt = np.clip((a8.astype(np.float64) / 255.0) * float(style_opacity), 0.0, 1.0) + coverage = np.zeros(a_pt.shape, dtype=np.float64) + saturated = a_pt >= 1.0 + partial = ~saturated & (a_pt > 0.0) + coverage[partial] = -np.expm1(counts[partial] * np.log1p(-a_pt[partial])) + coverage[saturated] = 1.0 + alpha = (np.clip(coverage, 0.0, 1.0) * 255.0).astype(np.uint8) + alpha[(counts <= 0) | (a8 == 0)] = 0 + return alpha + + def _density_image( d: dict, blob: bytes, cols: list, sx: _Scale, sy: _Scale, style: dict, svg: _Svg ) -> str: @@ -2577,18 +2601,23 @@ def _density_image( grid = _density_column(blob, cols[d["buf"]], d).reshape(h, w) gmax = float(d.get("max") or 1.0) or 1.0 tnorm = np.clip(grid / gmax, 0.0, 1.0) - paint_alpha: float | np.ndarray = 1.0 if d.get("rgba") is not None: - # Mean point color per cell (LOD doc §2): rgb from the shipped plane, - # count drives only the alpha ramp, scaled by the cell's mean point - # alpha — the same law as the client's DENSITY_FS. + # Mean point color per cell (LOD doc §2): rgb from the shipped plane; + # displayed alpha is the PHYSICAL compositing of the cell's points — + # 1 − (1 − a_pt)^count for drawn per-point alpha a_pt = channel alpha + # × style opacity (folded INSIDE the exponent: dense cells saturate + # past the style opacity exactly like overplotted marks). Same law as + # the client's texture upload. meta = cols[d["rgba"]] mean = np.frombuffer( blob, dtype=np.uint8, count=meta["len"], offset=meta["byte_offset"] ).reshape(h, w, 4) rgb = mean[..., :3] - paint_alpha = mean[..., 3].astype(np.float64) / 255.0 - elif d.get("color") is not None: + alpha = _physical_density_alpha(grid, mean[..., 3], _fill_opacity(style, 0.85)) + rgba = np.dstack([rgb, alpha])[::-1].tobytes() # flip: PNG rows are top-first + return _grid_image(w, h, rgba, d["x_range"], d["y_range"], sx, sy) + paint_alpha: float = 1.0 + if d.get("color") is not None: red, green, blue, alpha8 = _paint_rgba8(d["color"]) rgb = np.empty((h, w, 3), dtype=np.uint8) rgb[:] = (red, green, blue) diff --git a/python/xy/_trace.py b/python/xy/_trace.py index 73629a7d..b3b82c33 100644 --- a/python/xy/_trace.py +++ b/python/xy/_trace.py @@ -71,6 +71,14 @@ class Trace: # subset is dropped instead of translating indices in the wrong space # (§16/§17: exact readout beats stale availability). drill_seq: int = 0 + # Recent shipped subsets, {drill_seq: sel}, bounded to DRILL_HISTORY_KEEP + # (LOD doc T13): the client may pick against a retired cached point window + # whose seq is no longer current — translating through the remembered + # subset keeps that hover exact instead of dead. Cleared on data changes + # (an old subset's indices would then name different rows — §16). + drill_history: dict[int, Any] = field( + default_factory=dict, init=False, repr=False, compare=False + ) # Count-pyramid cache (§5 Tier 3), managed by `interaction.py`: None = # never tried, 0 = tried and not applicable, otherwise the native handle. # The finalizer frees the native side when the trace is collected. diff --git a/python/xy/config.py b/python/xy/config.py index 60dd4310..05cf633b 100644 --- a/python/xy/config.py +++ b/python/xy/config.py @@ -24,8 +24,9 @@ # Scatter above this many points switches to Tier-2 density aggregation (§5): # instead of shipping/drawing every point (fill-rate + the ~1 GB single-alloc # cliff, §5 F3), the kernel bins the viewport into a density grid the client -# draws with the trace's own colors — count drives only the alpha (LOD doc -# §2). Screen-bounded transport and VRAM regardless of point count. +# draws with the trace's own colors, composited at the points' own alpha +# (LOD doc §2; count-only surfaces keep the log count ramp). Screen-bounded +# transport and VRAM regardless of point count. SCATTER_DENSITY_THRESHOLD = 200_000 # Absolute direct-draw ceiling; above this, density is forced even if the user @@ -64,12 +65,41 @@ # static and re-ship large; a few points per cell keeps drill-out continuous. DENSITY_TARGET_POINTS_PER_CELL = 16.0 -# Hybrid density overlay (§5): when scatter is aggregated, ship a small, -# deterministic sample of real points over the density texture. This keeps -# zoomed-out views from becoming pure heatmaps while staying payload-bounded. +# Deterministic point sample retained with the FIRST density payload only +# (§28/#225): interactive density_view replies ship no samples at all — the +# density surface already wears the data's own colors (LOD doc §2), and real +# points arrive the moment a window fits the budget. Kernel-attached clients +# never DRAW the retained sample either (a fixed-size sample reads as +# individual data points at zooms where real points are sub-pixel or one +# request away); they use it CPU-side as the distribution-true estimator for +# the points-band request gate (LOD doc T13, `lodSampleViewCount`). The +# standalone (kernel-less) client keeps it as its re-bin worker's CPU source +# and draws it below the resolvable-count gate, where it is the only point +# representation that build will ever have. DENSITY_SAMPLE_TARGET = 8_192 DENSITY_SAMPLE_SEED = 0 +# Padded drill windows (LOD doc T13): a points-tier reply ships the largest +# ALIGNED window around the view whose exact count still fits the budget, so +# the client's window cache answers nearby pans/zooms with zero round-trips. +# Ladder of padded-span targets (× the view span), coarsest first; bounds snap +# outward to a power-of-two grid over the trace's extent (lod.aligned_window), +# making consecutive pans resolve to the SAME window. +DRILL_PAD_TARGETS = (8.0, 4.0, 2.0) +# Hard per-axis cap on the padded span (× the view span). Must stay well under +# the client's §16 re-encode bound (1/256 of the window span): the shipped +# offset encoding centers on the padded window, and a window unboundedly wider +# than the view would let deep zooms outrun f32 precision before the re-encode +# request re-arms. +DRILL_PAD_SPAN_CAP = 64.0 + +# Recent drilled subsets kept resolvable for picks (LOD doc T13): the client +# may serve a view from a retired cached point window whose drill_seq is no +# longer current; translating through the remembered subset keeps hover exact +# instead of dead. Bounded — anything older resolves to None, never to a +# wrong row (§16 exact-or-nothing). +DRILL_HISTORY_KEEP = 8 + # CVD-safe default categorical palette (§20/§36 default theme). Eight slots in # a fixed order; charts render on unknown host surfaces, so every step sits in # the OKLCH lightness band both light and dark modes share (L 0.48–0.67) and is diff --git a/python/xy/interaction.py b/python/xy/interaction.py index 114d2616..feb3b140 100644 --- a/python/xy/interaction.py +++ b/python/xy/interaction.py @@ -24,10 +24,10 @@ from .config import ( DECIMATION_THRESHOLD, DEFAULT_PALETTE, - DENSITY_SAMPLE_SEED, - DENSITY_SAMPLE_TARGET, DENSITY_TARGET_POINTS_PER_CELL, # noqa: F401 (historic import path) DRILL_EXIT_FACTOR, + DRILL_PAD_SPAN_CAP, + DRILL_PAD_TARGETS, PYRAMID_BASE_DIM, PYRAMID_MAX_DIM, PYRAMID_MIN_POINTS, @@ -95,8 +95,16 @@ def pick( except ValueError: return None if dseq is not None and dseq != t.drill_seq: - return None - shipped_sel = _point_shipped_sel(t) + # The client may pick against a RETIRED cached point window (LOD doc + # T13) whose subset version is no longer the current drill. Recent + # subsets stay resolvable through the bounded history; anything older + # (or bumped by exits/data changes) drops the pick rather than reading + # the index in the wrong subset space (§16 exact-or-nothing). + shipped_sel = lod.drill_history(t, dseq) + if shipped_sel is None: + return None + else: + shipped_sel = _point_shipped_sel(t) if shipped_sel is not None: if idx < 0 or idx >= len(shipped_sel): return None @@ -484,74 +492,89 @@ def _decode_log_u8(buf: bytes, gmax: float) -> np.ndarray: return np.expm1((v / 255.0) * np.log1p(gmax)) -def _density_sample_update( +# Interactive density replies deliberately ship NO point sample (#225): a +# fixed-size sample above the drill budget reads as individual data points at +# a zoom where real points are sub-pixel — misrepresenting the dataset — and +# the density surface already wears the data's own colors (LOD doc §2). Real +# points arrive the moment a window fits the budget. The only retained sample +# is the first-payload one (`_payload._density_sample_spec`), which the client +# draws solely below the resolvable-count gate and the standalone re-bin +# worker keeps as its CPU source. + + +def _pyramid_source_shape( + t: Any, lo_x: float, hi_x: float, lo_y: float, hi_y: float +) -> tuple[int, int] | None: + """The pyramid's finest-level cell budget under a window, per axis. + + Wire economy (§29 / #225 follow-up): composing a pyramid-served window to + full screen resolution upsamples blocky base cells into a grid several + times larger than the information it carries — a ~2.7 MB reply whose + content is a few hundred KB of source cells. Clamping the composed grid + to `(cells under the window at the finest level) + 1` per axis ships the + same detail (the client's own texture filtering reproduces the upscale) + at a fraction of the bytes. + """ + base = int(getattr(t, "_pyr_base_dim", 0) or PYRAMID_BASE_DIM) + ex0, ex1, ey0, ey1 = t.x.min, t.x.max, t.y.min, t.y.max + span_x, span_y = ex1 - ex0, ey1 - ey0 + if not all(np.isfinite(v) for v in (ex0, ex1, ey0, ey1)) or span_x <= 0 or span_y <= 0: + return None + frac_x = min(1.0, max(0.0, (hi_x - lo_x) / span_x)) + frac_y = min(1.0, max(0.0, (hi_y - lo_y) / span_y)) + return max(1, math.ceil(base * frac_x) + 1), max(1, math.ceil(base * frac_y) + 1) + + +def _padded_drill_window( fig: "Figure", t: Any, - sel: np.ndarray, - visible: int, + pyr: int | None, lo_x: float, hi_x: float, lo_y: float, hi_y: float, - writer: lod.BufferWriter, -) -> Optional[dict[str, Any]]: - if visible <= 0: +) -> Optional[tuple[float, float, float, float, np.ndarray]]: + """The widest aligned window around the view that still drills (T13). + + The client elides any request whose view an exact shipped window already + contains (T12) and caches retired windows, so every extra span this window + can afford converts future pans and zooms into zero-round-trip renders. + Bounds snap outward to the power-of-two grid over the trace's extent + (`lod.aligned_window`), making consecutive pans resolve to the SAME + window; the coarsest ladder rung whose exact in-window count fits the + budget wins. Returns None (drill the raw view window) when padding buys + nothing: nonlinear axes (raw-space alignment mis-sizes log windows near + zero), non-finite extents, or every rung over budget. + """ + if fig._axis_scale(t.x_axis) != "linear" or fig._axis_scale(t.y_axis) != "linear": return None - categories = None - if t.color_ch and t.color_ch.mode == "categorical" and t.color_ch.codes is not None: - categories = t.color_ch.codes[sel] - sample_sel = lod.sample_rows_for_target( - sel, - DENSITY_SAMPLE_TARGET, - categories=categories, - seed=DENSITY_SAMPLE_SEED, - ) - if len(sample_sel) == 0: + ex0, ex1, ey0, ey1 = t.x.min, t.x.max, t.y.min, t.y.max + if not all(np.isfinite(v) for v in (ex0, ex1, ey0, ey1)): return None - xs, ys = t.x.values[sample_sel], t.y.values[sample_sel] - x_ref, y_ref = lod.add_window_xy( - writer, - xs, - ys, - lo_x, - hi_x, - lo_y, - hi_y, - fig._axis_scale(t.x_axis), - fig._axis_scale(t.y_axis), - ) - color_spec, size_spec = fig._ship_channels( - t, sample_sel, writer.add_f32, writer.add_u8, quantize_continuous=True - ) - style = dict(t.style) - try: - style["opacity"] = min(float(style.get("opacity", 0.8)), 0.55) - except (TypeError, ValueError): - style["opacity"] = 0.55 - sample = { - "mode": "sampled", - "n": int(len(sample_sel)), - "visible": int(visible), - "target": DENSITY_SAMPLE_TARGET, - "level": 0, - "seed": DENSITY_SAMPLE_SEED, - "x": x_ref, - "y": y_ref, - "x_range": [lo_x, hi_x], - "y_range": [lo_y, hi_y], - "color": color_spec, - "size": size_spec, - "style": style, - } - if t.stroke_ch is not None: - sample["stroke"] = channels.ship_color_channel( - t.stroke_ch, sample_sel, writer.add_f32, writer.add_u8, DEFAULT_PALETTE - ) - if t.style_channels: - sample["channels"] = channels.ship_style_channels( - t.style_channels, sample_sel, writer.add_f32, writer.add_u8 - ) - return sample + span_x, span_y = hi_x - lo_x, hi_y - lo_y + budget = SCATTER_DENSITY_THRESHOLD + for pad in DRILL_PAD_TARGETS: + px0, px1 = lod.aligned_window(lo_x, hi_x, ex0, ex1, pad) + py0, py1 = lod.aligned_window(lo_y, hi_y, ey0, ey1, pad) + if px0 == lo_x and px1 == hi_x and py0 == lo_y and py1 == hi_y: + continue + # §16 precision guard: the shipped offset encoding centers on THIS + # window, and the client re-requests precision only below 1/256 of the + # window span — a window unboundedly wider than the view (tiny view + # over a small dataset) would let deep zooms outrun f32 first. + if px1 - px0 > span_x * DRILL_PAD_SPAN_CAP or py1 - py0 > span_y * DRILL_PAD_SPAN_CAP: + continue + if pyr is not None: + # Cheap center-in-cell estimate gates the exact O(N) verify scan; + # the margin keeps a boundary-straddling estimate from wasting a + # scan on a window the exact count then rejects. + est = kernels.pyramid_count(pyr, px0, px1, py0, py1) + if est is not None and est > budget * 0.85: + continue + sel = kernels.range_indices(t.x.values, t.y.values, px0, px1, py0, py1) + if len(sel) <= budget: + return px0, px1, py0, py1, sel + return None def _quantize_dval(dval: np.ndarray) -> np.ndarray: @@ -699,23 +722,30 @@ def density_view( False, aggregate_reduction="pyramid-count", ) + # Wire economy: never compose more grid cells than the finest + # level resolves under this window — a full-screen grid of + # upsampled base cells is the same picture at several times the + # bytes (the client's texture filtering does the upscale). + gw, gh = plan.grid_w, plan.grid_h + source = _pyramid_source_shape(t, lo_x, hi_x, lo_y, hi_y) + if source is not None: + gw = max(16, min(gw, source[0])) + gh = max(16, min(gh, source[1])) # Colored pyramids compose the mean-color plane with the counts # (same level, same max_upsample); both refusals (outresolved # window, missing planes) fall through to the paths below. if getattr(t, "_pyr_colored", False): res_color = kernels.pyramid_compose_color( - pyr, lo_x, hi_x, lo_y, hi_y, plan.grid_w, plan.grid_h, max_upsample + pyr, lo_x, hi_x, lo_y, hi_y, gw, gh, max_upsample ) res = (res_color[0], res_color[2]) if res_color is not None else None rgba_grid = res_color[1] if res_color is not None else None else: - res = kernels.pyramid_compose( - pyr, lo_x, hi_x, lo_y, hi_y, plan.grid_w, plan.grid_h, max_upsample - ) + res = kernels.pyramid_compose(pyr, lo_x, hi_x, lo_y, hi_y, gw, gh, max_upsample) if res is not None: grid, level = res visible = plan.visible - w, h = plan.grid_w, plan.grid_h + w, h = gw, gh binning = f"pyramid-L{level}{'-upsampled' if no_rescan and level == 0 else ''}" lod.exit_drill(t) # Tier-3 spatial index: when the pyramid can only serve this window blurry @@ -790,7 +820,17 @@ def density_view( plan = lod.plan_view_lod(request, len(sel), SCATTER_DENSITY_THRESHOLD, t.drill_mode) visible = plan.visible if plan.exact: - return _drill_points(fig, t, sel, visible, lo_x, hi_x, lo_y, hi_y, w, h) + # Ship the widest aligned window that still fits the budget (T13) + # so the client's point-window cache answers nearby pans/zooms + # locally; the raw view window is the floor. `visible` describes + # the SHIPPED window; the view's own count keeps driving the + # density→points intensity handoff. + padded = _padded_drill_window(fig, t, pyr, lo_x, hi_x, lo_y, hi_y) + if padded is not None: + lo_x, hi_x, lo_y, hi_y, sel = padded + return _drill_points( + fig, t, sel, len(sel), lo_x, hi_x, lo_y, hi_y, w, h, blend_visible=visible + ) lod.exit_drill(t) w, h = plan.grid_w, plan.grid_h @@ -812,11 +852,6 @@ def density_view( writer = lod.BufferWriter() density_wire, gmax = _encode_log_u8(grid) density_buf = writer.add_raw(density_wire) - sample = ( - _density_sample_update(fig, t, sel, visible, lo_x, hi_x, lo_y, hi_y, writer) - if binning == "exact" - else None - ) density = { "buf": density_buf, "w": w, @@ -835,8 +870,6 @@ def density_view( density["color_agg"] = "mean" if t.color_ch and t.color_ch.mode == "constant" and t.color_ch.constant is not None: density["color"] = t.color_ch.constant - if sample is not None: - density["sample"] = sample return ( { "traces": [ @@ -866,6 +899,7 @@ def _drill_points( hi_y: float, w: int, h: int, + blend_visible: int | None = None, ) -> tuple[dict[str, Any], list[bytes]]: """Ship the visible subset of a Tier-2 scatter as real points (§5 drill-in). @@ -873,10 +907,14 @@ def _drill_points( ship in the direct-scatter wire shape, normalized over their *global* domain so colors/sizes stay stable across views; offsets re-center on the window midpoint (§16); each point carries its local log-density plus a - `lod_blend` weight (visible/budget) so the density→points handoff stays - continuous (§5). The surface wears the mean point color (LOD doc §2), so - the handoff is intensity-only: fresh marks enter at their cell's - count-alpha and ease to native opacity, with hue continuous throughout.""" + `lod_blend` weight so the density→points handoff stays continuous (§5). + The window may be a padded aligned superset of the requested view (T13), + so `visible` counts the SHIPPED window while `blend_visible` — the + requested view's own count — drives the handoff weight: the intensity a + user sees at the swap belongs to the view, not to padding they can't see. + The surface wears the mean point color (LOD doc §2), so the handoff is + intensity-only: fresh marks enter at their cell's count-alpha and ease to + native opacity, with hue continuous throughout.""" xs, ys = t.x.values[sel], t.y.values[sel] writer = lod.BufferWriter() x_ref, y_ref = lod.add_window_xy( @@ -909,8 +947,11 @@ def _drill_points( # 1.0 right at the boundary → points enter at the density surface's # local count-alpha; →0 as zoom deepens. The surface wears the mean point # color (LOD doc §2), so marks arrive in their native colors and only - # intensity eases. - lod_blend = float(min(1.0, visible / SCATTER_DENSITY_THRESHOLD)) + # intensity eases. Keyed on the VIEW's count when the window is padded — + # padding widens what ships, not what the user is looking at. + lod_blend = float( + min(1.0, (visible if blend_visible is None else blend_visible) / SCATTER_DENSITY_THRESHOLD) + ) trace_update = { "id": t.id, "mode": "points", @@ -1146,6 +1187,10 @@ def _style_tail(name: str, values: Any) -> Optional[np.ndarray]: # "not applicable" result; let the next wide view build it. t._pyr_handle = None lod.exit_drill(t) + # Remembered subsets were computed against the pre-append canonical state; + # the client rebuilds its GPU traces (and drops its cached point windows) + # on the refresh anyway, so stale-seq picks must die rather than translate. + lod.clear_drill_history(t) # Split layout, same as first paint (§29): per-column borrowed views, no # join copy. The spec itself names the append — `append.seq` is the apply diff --git a/python/xy/lod.py b/python/xy/lod.py index 5a8e6675..35d0f981 100644 --- a/python/xy/lod.py +++ b/python/xy/lod.py @@ -27,7 +27,12 @@ import numpy as np from . import kernels -from .config import DENSITY_TARGET_POINTS_PER_CELL, DRILL_EXIT_FACTOR, MAX_SCREEN_DIM +from .config import ( + DENSITY_TARGET_POINTS_PER_CELL, + DRILL_EXIT_FACTOR, + DRILL_HISTORY_KEEP, + MAX_SCREEN_DIM, +) _SPLITMIX_INCREMENT = np.uint64(0x9E3779B97F4A7C15) _SPLITMIX_MUL_1 = np.uint64(0xBF58476D1CE4E5B9) @@ -720,19 +725,78 @@ def enter_drill(trace: Any, sel: np.ndarray) -> int: trace.drill_mode = True trace.shipped_sel = sel trace.drill_seq += 1 + # Remember recent subsets (T13): the client can serve a view from a + # RETIRED cached point window whose drill_seq is no longer current, and a + # pick against it should still translate exactly. Bounded FIFO — an + # expired seq resolves to None (a dropped pick), never to a wrong row. + history = trace.drill_history + history[trace.drill_seq] = sel + while len(history) > DRILL_HISTORY_KEEP: + del history[next(iter(history))] return trace.drill_seq def exit_drill(trace: Any) -> None: """Back to the aggregate: no per-point marks, no pick mapping. Bumps the version when leaving an actual drill so a drilled-index pick arriving late - is rejected instead of being read as a *canonical* index.""" + is rejected instead of being read as a *canonical* index. Remembered + subsets survive the exit — client-side cached point windows outlive the + kernel's current-tier choice (T13) — until a data change clears them.""" if trace.drill_mode: trace.drill_seq += 1 trace.drill_mode = False trace.shipped_sel = None +def drill_history(trace: Any, seq: int) -> np.ndarray | None: + """The shipped subset a recent `drill_seq` named, or None when expired. + + None must stay None at the call site (drop the pick): translating through + the wrong subset would read an arbitrary canonical row (§16).""" + return trace.drill_history.get(seq) + + +def clear_drill_history(trace: Any) -> None: + """Forget remembered subsets after a data change: the retained indices + were computed against the previous canonical state, and a client window + built on them is rebuilt anyway (append/update rebuilds GPU traces).""" + trace.drill_history.clear() + + +def aligned_window( + lo: float, hi: float, extent_lo: float, extent_hi: float, pad: float +) -> tuple[float, float]: + """Snap a 1-D window outward to the power-of-two grid over its extent. + + The grid level is a pure function of the extent and the window's span + bucket: one block spans ``extent / 2**level``, the coarsest level with a + block no wider than ``pad × span``, and the window's bounds snap outward + to block edges. Every request whose span falls in the same power-of-two + bucket therefore resolves to the SAME aligned bounds regardless of pan + position — full-point buffers aligned by dimension, so client caches can + key, dedupe, and reuse them (LOD doc T13). The result always CONTAINS the + input window (client request elision needs containment, and a view panned + past the data extent must stay contained even though nothing lives there); + the snapped span stays within ``(1 + 2·pad) × span`` of the input window. + """ + span = hi - lo + extent = extent_hi - extent_lo + if not ( + np.isfinite(span) and np.isfinite(extent) and span > 0.0 and extent > 0.0 and pad >= 1.0 + ): + return lo, hi + if pad * span >= extent: + return min(extent_lo, lo), max(extent_hi, hi) + level = max(0, math.ceil(math.log2(extent / (pad * span)))) + block = extent / (1 << level) + b0 = math.floor((lo - extent_lo) / block) + b1 = math.ceil((hi - extent_lo) / block) + # The min/max guards absorb the one-ulp case where a bound lands exactly + # on a grid line and float rounding would nudge the snapped edge inside + # the window — containment is the contract, alignment the optimization. + return (min(lo, extent_lo + b0 * block), max(hi, extent_lo + b1 * block)) + + class BufferWriter: """Accumulates a view-update's binary buffers (typed scalars, never JSON numbers). The update spec references entries by index — the same shape diff --git a/python/xy/marks.py b/python/xy/marks.py index 33463875..aad4d409 100644 --- a/python/xy/marks.py +++ b/python/xy/marks.py @@ -1464,7 +1464,7 @@ def scatter( ) mean_color_note = ( " The color channel is kept as the surface's per-cell mean point color" - " (count drives the alpha)." + " (composited at the points' own alpha)." if color_aggregates else "" ) diff --git a/spec/design-dossier.md b/spec/design-dossier.md index f423d6f0..07c0068e 100644 --- a/spec/design-dossier.md +++ b/spec/design-dossier.md @@ -416,9 +416,15 @@ re-exports several of them as a historic import path and is not listed for those | `MAX_CONTOUR_WORK` | `4_000_000` | Ceiling on contour `cells × levels`; a request over it raises instead of allocating an unbounded segment buffer. | `marks.py`, `_native.py` | | `DRILL_EXIT_FACTOR` | `1.15` | Hysteresis multiplier on the drill boundary: a trace already drilled to real points stays drilled until the visible count exceeds `budget × 1.15`. | `lod.py` (`drill_decision`, `plan_view_lod`), `interaction.py`; mirrored as `LOD_DRILL_EXIT_FACTOR` in `js/src/45_lod.ts` | | `DENSITY_TARGET_POINTS_PER_CELL` | `16.0` | Target points per cell when sizing an aggregation grid, so a barely-over-budget view does not get a one-point-per-pixel grid that looks like static and re-ships large. | `lod.py` | -| `DENSITY_SAMPLE_TARGET` | `8_192` | Size of the deterministic real-point sample shipped over an aggregated scatter's density texture (hybrid overlay). | `_payload.py`, `interaction.py` | -| `DENSITY_SAMPLE_SEED` | `0` | Seed for that sample; a fixed seed makes the overlay identical across re-ships of the same view. | `_payload.py`, `interaction.py` | -| `LOD_SAMPLE_FADE_COVER_HI` / `LOD_SAMPLE_FADE_COVER_LO` | `1/4` / `1/32` | Fallback bound on the hybrid-overlay sample (T9): overlays ride their density windows and the drawn one is the best cached window covering the view (full alpha), so the band only governs a view NO cached window covers — the best partial draws at full alpha while its window covers ≥ 1/4 of the view area, hidden below 1/32, log-eased between. The band value is a *composited* opacity target (per-point alpha solved against the expected overplot, `1−(1−band)^(1/k)`), so a partial overlay can never overplot into a false cluster. | `js/src/45_lod.js` (`lodSampleViewAlpha`, `lodSampleForView`) | +| `DENSITY_SAMPLE_TARGET` | `8_192` | Size of the deterministic real-point sample retained with the FIRST density payload only (#225: interactive density replies ship no samples; the client draws the retained overlay solely below the T9 resolvable-count gate, and the standalone re-bin worker keeps it as its CPU source). | `_payload.py` | +| `DENSITY_SAMPLE_SEED` | `0` | Seed for that sample; a fixed seed makes the overlay identical across re-ships of the same view. | `_payload.py` | +| `LOD_SAMPLE_FADE_COVER_HI` / `LOD_SAMPLE_FADE_COVER_LO` | `1/4` / `1/32` | Fallback bound on the retained sample overlay (T9): overlays ride their density windows and the drawn one is the best cached window covering the view (full alpha), so the band only governs a view NO cached window covers — the best partial draws at full alpha while its window covers ≥ 1/4 of the view area, hidden below 1/32, log-eased between. The band value is a *composited* opacity target (per-point alpha solved against the expected overplot, `1−(1−band)^(1/k)`), so a partial overlay can never overplot into a false cluster. Every candidate first passes the #225 resolvability gate (estimated in-view count ≤ the direct budget). | `js/src/45_lod.ts` (`lodOverlayResolvable`, `lodSampleViewAlpha`, `lodSampleForView`) | +| `DRILL_PAD_TARGETS` | `(8, 4, 2)` | Ladder of padded-span targets (× the view span) for points-tier replies: the coarsest ALIGNED window around the view whose exact count fits the budget ships, so the client's point-window cache answers nearby pans/zooms with zero round-trips (LOD doc T13). | `interaction.py` (`_padded_drill_window`), `lod.py` (`aligned_window`) | +| `DRILL_PAD_SPAN_CAP` | `64.0` | Hard per-axis cap on a padded drill window's span (× the view span), kept well under the client's §16 re-encode bound (1/256 of window span) so deep zooms can always re-tighten the f32 offset encoding. | `interaction.py` | +| `DRILL_HISTORY_KEEP` | `8` | Recent drilled subsets kept resolvable per trace, so picks against a retired cached point window (T13) still translate exactly; older seqs drop the pick, data changes clear the history. | `lod.py` (`enter_drill`, `drill_history`), `interaction.py` (`pick`) | +| `LOD_POINT_CACHE_WINDOWS` | `3` | Retired exact point windows kept per trace client-side beyond the live drill; LRU-bounded VRAM, swept by the T11 outgrown rule. | `js/src/45_lod.ts` (`lodRetireDrill`, `lodPromoteCachedDrill`) | +| `LOD_POINTS_REQUEST_BAND` | `4` | The aggregate tier never refines per view (T13, revised): a raw-view `density_view` goes out only when the estimated in-view count sits within `budget × 4` of points territory — the LOWER of an area-scaled cached-window count and the retained sample counted in-view (`lodSampleViewCount`, distribution-true where area-scaling over-estimates sparse tails). | `js/src/45_lod.ts` (`lodAggregateStands`) | +| `LOD_AGG_STEP_FACTOR` / `LOD_AGG_STEP_MAX` / `LOD_AGG_STEP_SLACK` | `4` / `2` / `1.5` | The stepped aggregate ladder (T13): while standing, the only density request is the view snapped outward to a power-of-4 block grid over the extent (per axis), at most 2 steps below home, and only when every covering texture is coarser than the step by more than the slack. Quantized windows are pan-stable and dedupable — at most 2 smooth-to-smooth swaps before points, worst-case softness ≈ 4× stretch per axis. | `js/src/45_lod.ts` (`lodAggregateStepWindow`) | | `DEFAULT_PALETTE` | 10 CVD-safe hex entries | Per-trace default color cycle and the fallback categorical palette when a channel supplies none (§20/§36). | `marks.py`, `_payload.py`, `_svg.py`, `_raster.py` | | `PYRAMID_MIN_POINTS` | `2_000_000` | Trace size at/above which a Tier-3 tile pyramid is built lazily; smaller traces never pay for one. | `interaction.py` | | `PYRAMID_BASE_DIM` | `2048` | Edge of the pyramid's base level in cells (`dim²` u32 counts, ~1/3 overhead for the coarser levels); sets resident pyramid bytes. | `interaction.py` | @@ -466,8 +472,9 @@ F3, still pending (above). - **WebGPU primary, WebGL2 fallback.** One ``, everything is GPU primitives. - **Instanced draws** for markers/bars/lines — one draw call for millions of marks. - **Density textures** for aggregated tiers (§5, Tier 2) — mean point color per - cell, count as alpha (LOD doc §2); premultiplied RGBA8 upload for - channel-bearing traces, tinted R8 count texture for constant-color ones. + cell, composited at the points' own alpha (`1−(1−a_pt)^count`, LOD doc §2); + premultiplied RGBA8 upload for channel-bearing traces, tinted R8 + log-count-ramp texture for constant-color ones. - **DOM/SVG only for chrome** — axis tick labels, legend, title, tooltip. Little of it, and it stays crisp/accessible/selectable. - **Retained scene graph**, spec-diff → buffer-diff. Pan/zoom is a view-matrix uniform @@ -1010,9 +1017,10 @@ means uniform in the axis's scale coordinates, not in raw data. Concretely: before `bin_2d`; the wire keeps raw `x_range`/`y_range` endpoints and the rule "cells are uniform in scale coordinates" (wire-protocol.md §3). Without this, clusters at x=1 and x=1000 share one cell on a symlog axis despite - being a third of the screen apart. The drill handoff's per-point + being a third of the screen apart. The count-only drill handoff's per-point `local_log_density` bins in the same space so its count-alpha matches the - grid's (hue already matches — the grid wears the mean point color, LOD doc §2). + grid's (mean-color surfaces need no handoff at all — they composite like + the points themselves, LOD doc §2 rules 1/5). - **The raw-space tile pyramid** (Tier 3) cannot compose a scale-coordinate grid, so traces on a nonlinear axis skip pyramid build/compose and always take the exact scan (`binning: "exact"`). Cost: the O(visible) rescan the diff --git a/spec/design/lod-architecture.md b/spec/design/lod-architecture.md index fff965c1..a83082cf 100644 --- a/spec/design/lod-architecture.md +++ b/spec/design/lod-architecture.md @@ -28,7 +28,7 @@ pixel-area or overdraw term exists in `python/xy/` or `js/src/` today. |---|---|---|---|---| | 0 | Direct | every visible mark, exact | O(visible) verts | shipped (all kinds) | | 1 | Shape-preserving reduction | per-pixel-column aggregate that IS the mark's meaning (M4 for lines; OHLC-bucket for candles when finance returns; max-bin for bars) | O(px) verts | shipped (line) | -| 2 | Density / aggregate surface | mean-point-color texture, count as alpha (§2) | O(screen) texels | shipped (scatter) | +| 2 | Density / aggregate surface | mean-point-color texture composited at the points' own alpha (§2) | O(screen) texels | shipped (scatter) | | 3 | Out-of-core tiles | Tier-2 pyramid where not all tiles are resident | O(visible tiles) | **this doc** | **Invariant L1 (recorded reduction):** every update carries `tier`, `mode`, @@ -44,7 +44,7 @@ real marks with real channels — never a sample, never an aggregate. | Kind | Tier 0 | Tier 1 | Tier 2 | zoom-in recovery | |---|---|---|---|---| -| scatter | points | — (unordered decimation lies, §28) | mean-color density grid (§2; count planes + color planes, §4 below) | drill-in ships exact visible points + channels (shipped); deeper zooms inside the exact window are answered locally, no request (T12) | +| scatter | points | — (unordered decimation lies, §28) | mean-color density grid (§2; count planes + color planes, §4 below) | drill-in ships exact points + channels for a padded ALIGNED window (T13); views inside any shipped window — live or cached — are answered locally, no request (T12/T13) | | line/area | polyline | M4 per column (extrema-exact) | — (M4 already screen-bounded) | re-decimate window (shipped) | | heatmap | cell rects | — | mip-style tile reduction of the user grid (max/mean recorded) | finer pyramid level, then exact cells | | histogram/bar | rects | re-bin at viewport resolution | — (bins are already aggregates) | re-bin visible window (finer bins = more truth, never less) | @@ -94,19 +94,33 @@ metadata, or encoded-buffer assembly. Aggregating positions is easy; aggregating **color/size/category** without lying is the hard part (§5-F5). The governing principle, shipped: **the -density surface wears the data's own colors; count drives only the alpha.** -A cell's color is what the eye would see if every one of its points were -drawn sub-pixel — the physically downsampled image — so the aggregate view -and the point view are two magnifications of the same picture, not two -different charts. More points → deeper (more opaque) color; fewer points → -lighter; the color itself always comes from the represented points. - -1. **Count is always channel 0, and it is the ALPHA channel.** The count - grid ships log-tone-mapped (`buf` + `max`, log-u8) and drives opacity - exclusively. Color rides on top of count, never instead of it — - otherwise sparse-but-extreme cells look as important as dense ones. No - shipped path colormaps counts anymore (the client keeps a count→LUT - fallback solely for hand-built/legacy count-only specs). +density surface wears the data's own colors, and it composites like the +data's own points.** A cell is what the eye would see if every one of its +points were drawn sub-pixel — the physically downsampled image — so the +aggregate view and the point view are two magnifications of the same +picture, not two different charts: same hue (the mean point color), same +lightness (the points' own alpha compositing). + +1. **A mean-color cell's displayed alpha is the PHYSICAL compositing of its + own points:** `1 − (1 − a_pt)^k` for k points whose drawn per-point + alpha is `a_pt = mean channel alpha × trace style opacity` — exactly + what k overplotted marks composite to, saturating after a few points + just as real marks do. Style opacity folds INSIDE the exponent (dense + cells saturate past it exactly like real overplot), so the draw path + applies only transition fades on top of these textures — never the + style opacity again. No window normalization touches the alpha, so + lightness cannot swing between windows or across the texture↔points + boundary (the previous law — a per-window log-count tone curve driving + opacity — did both: a field capture of the 100M drilldown showed the + surface visibly lighter/darker than the very points it aggregated). The + count grid still ships (`buf` + `max`, log-u8): it marks occupancy, is + the compositing exponent, and remains the ONLY structure — and the alpha + ramp — for count-only surfaces (constant-color traces, hand-built/legacy + specs), which keep the log tone curve and its eased normalization (T4). + The recorded cost of physical alpha: density beyond a few points per + cell no longer reads in the surface (real overplotted points saturate + identically — the aggregate is exactly as saturated as the truth it + downsamples; drill-in is where per-point structure returns). 2. **Per-point colors → per-cell alpha-weighted MEAN of the *resolved* colors, averaged in linear light.** One law for every channel mode — continuous (colormapped value), categorical (palette color), and @@ -115,9 +129,10 @@ lighter; the color itself always comes from the represented points. pipeline: checked-in sRGB⇄linear-u16 tables, u64 sums — bitwise deterministic across thread counts and platforms), then quantized back to sRGB. Ships as a `w*h*4` straight-alpha RGBA8 plane (`rgba`, mean color + - mean point alpha) with `color_agg: "mean"` recorded; displayed alpha = - count tone-curve × mean point alpha. A cell of one point shows that - point's exact color; an all-invisible (alpha-0) cell never invents one. + mean point alpha) with `color_agg: "mean"` recorded; displayed alpha + follows rule 1 (physical compositing of ā over the cell count). A cell + of one point shows that point's exact color at that point's own alpha; + an all-invisible (alpha-0) cell never invents one. Constant-color traces ship no plane — the mean of a constant IS the constant, so the wire keeps the 1-byte count grid plus `color` and the client tints (bit-equivalent, 4× cheaper). Cost, recorded: a @@ -146,13 +161,16 @@ lighter; the color itself always comes from the represented points. `channels_dropped: true` + `dropped_channels` naming it. Color is *not* listed there when it aggregates — `color_agg: "mean"` records the transform instead (aggregated ≠ dropped). -5. **The drill handoff is intensity-only now:** hue is continuous by - construction (both sides show the points' colors), so freshly drilled - points arrive in their native colors and only their opacity hands off — - entering at the cell's count-alpha (`density_val` through the same tone - curve as the texture) and easing to native opacity as `lod_blend` → 0. - The same rule runs outbound: exiting marks re-target the aggregate's - count-alpha and melt into a surface that already matches their hue. +5. **Mean-color surfaces need no drill handoff at all:** hue is continuous + by construction (both sides show the points' colors) and, under rule + 1's physical alpha, so is lightness — the texture composites exactly + like the marks replacing it. Drilled points therefore enter and exit at + NATIVE opacity; the entry/exit alpha fades (T2) alone carry the swap, + and the client ignores `density_val`/`lod_blend` on such traces (the + wire still carries them — count-only surfaces keep the intensity + handoff: marks enter at the cell's count-alpha through the same tone + curve as their texture and ease to native as `lod_blend` → 0, and + exiting marks re-target it to melt into the ramp). **Invariant L4 (badge):** any active reduction that drops or transforms a channel renders a visible "aggregated: mean/…" badge sourced from the spec @@ -311,20 +329,24 @@ invariants so future kinds don't regress them: the aggregate→marks transition only — restarting per refresh reads as flashing; exit fade with the "dying" state so buffers outlive the fade). - **T3 — color-continuous:** the two sides of a transition display the same - statistic at the boundary. Hue is continuous *by construction* — the - aggregate surface wears the mean point color (§2), so marks and texture - agree on color at every zoom — and intensity hands off via the lod_blend - count-alpha ramp. The kernel's blend weight (`visible/budget`) is only ≈1 - when the swap happens at the budget boundary — a fast zoom skips levels - and lands marks with a mostly-native weight, a visible intensity pop at - the swap (live-drilldown field capture). The BOUNDARY is therefore the - transition itself, client-side: fresh marks arrive with the shown blend - seeded at 1 (entering at the aggregate's local count-alpha) and ease to - the kernel's native weight, and dying/exiting marks re-target blend 1 so - they melt into the texture as they fade (`lodApplyDrill`, - `lodBeginDrillExitContinuous`; revives restore the native weight via - `lodEnterDrillContinuous`). -- **T4 — normalization is eased, never stepped** (exposure-style normMax). + statistic at the boundary. On a mean-color surface BOTH channels are + continuous by construction — hue because the surface wears the mean point + color (§2 rule 2), lightness because it composites like the points + themselves (§2 rule 1) — so marks swap at native opacity with no + intensity handoff at all (§2 rule 5). Count-only surfaces keep the + lod_blend count-alpha ramp: the kernel's blend weight (`visible/budget`) + is only ≈1 when the swap happens at the budget boundary — a fast zoom + skips levels and lands marks with a mostly-native weight, a visible + intensity pop at the swap (live-drilldown field capture) — so the + BOUNDARY is the transition itself, client-side: fresh marks arrive with + the shown blend seeded at 1 (entering at the aggregate's local + count-alpha) and ease to the kernel's native weight, and dying/exiting + marks re-target blend 1 so they melt into the texture as they fade + (`lodApplyDrill`, `lodBeginDrillExitContinuous`; revives restore the + native weight via `lodEnterDrillContinuous`). +- **T4 — normalization is eased, never stepped** (exposure-style normMax) — + count-only surfaces; a mean-color texture's physical alpha is + max-independent, so it has no normalization to ease. - **T5 — stale replies die:** seq on view updates, drill_seq on subsets, pending-view hold for prefetched drills. - **T6 — invalid requests do not mutate:** malformed viewport/screen requests @@ -353,22 +375,34 @@ invariants so future kinds don't regress them: drive it out of the hold, so the drilled subset stayed painted and the full point cloud never returned. (Complements T7, which fixes the same visible symptom from the texture-lifetime side.) -- **T9 — the drawn sample describes the displayed window:** the deterministic - point sample shipped with an exact-scan density reply represents *its* - window only, so every sample overlay rides the density cache entry it was - computed for (`density.overlay`; replies that omit a sample — pyramid and - integral-image zoom-out paths, Phase 2 item 5 — simply add no overlay to - their entry). Each frame draws the overlay of the best cached window for - the current view (`lodSampleForView`, mirroring `lodDensityForView`): the - smallest window covering the view wins at full alpha, so a deep zoom-out - falls back through the cache to the HOME sample and the full point cloud - returns — points on screen always describe the window being displayed. - (The pre-pairing client retained one global sample and drew it whenever - the view merely overlapped its window: a drilled window's sample lingered - over every later zoom-out as an opaque "stuck point blob", pinned by a - live-drilldown wire capture — the blob's extent matched the last - sample-bearing reply's window exactly while the density surface kept - updating under it.) Only a view that NO cached window covers draws a +- **T9 — the drawn sample describes the displayed window, and only a + RESOLVABLE window (#225):** a sample overlay draws only when the view it + would describe could plausibly be points-tier — the overlay's recorded + window count, scaled by the view's share of its window, fits + `LOD_DIRECT_POINT_BUDGET` (`lodOverlayResolvable`). Above that, a + fixed-size sample reads as individual data points at a zoom where real + points are sub-pixel — sampling above the resolution of the graph + misrepresents the dataset (the #225 field capture: zooming out from a + drilled 100M-point cloud brought "individual points" back over the + aggregate) — and the density surface, which wears the data's own colors + (§2), stands alone. Interactive `density_view` replies therefore ship NO + sample at all: the only retained overlay is the first-payload one, which + doubles as the standalone re-bin worker's CPU source. A KERNEL-ATTACHED + client never draws it at any zoom (field follow-up): wherever a view is + resolvable the kernel ships REAL points, and a handful of retained sample + rows at full alpha there reads as data that isn't. The overlay is the + standalone (kernel-less) client's fallback — drawn only below the + resolvability gate, where it is the only point representation that build + will ever have. For the + overlays that DO draw, window pairing holds: every sample rides the + density cache entry it was computed for (`density.overlay`), and each + frame draws the overlay of the best cached window for the current view + (`lodSampleForView`, mirroring `lodDensityForView`) — the smallest window + covering the view wins at full alpha, so points on screen always describe + the window being displayed. (The pre-pairing client retained one global + sample and drew it whenever the view merely overlapped its window: a + drilled window's sample lingered over every later zoom-out as an opaque + "stuck point blob".) Only a view that NO cached window covers draws a partial overlay, faded by the window's share of the view area — full alpha at ≥ `LOD_SAMPLE_FADE_COVER_HI` (1/4), hidden at ≤ `LOD_SAMPLE_FADE_COVER_LO` (1/32), log-eased between @@ -379,11 +413,12 @@ invariants so future kinds don't regress them: near-opaque slab (a "fading" sample that never looked faded). The per-point alpha is solved as a = 1−(1−band)^(1/k), with k estimated from the drawn count, mean point footprint, and the window's on-screen area; - k ≤ 1 degenerates to the band value exactly. Selection and alpha are pure - functions of (view, cache): every zoom frame re-derives them, nothing - latches. Overlays die with their evicted cache entry (except the home/init - overlay, the standalone re-bin worker's CPU-side source), and the - "sampled n of N" badge reports the overlay actually drawn. + k ≤ 1 degenerates to the band value exactly. Selection, alpha, and the + resolvability gate are pure functions of (view, cache): every zoom frame + re-derives them, nothing latches. Overlays die with their evicted cache + entry (except the home/init overlay, the standalone re-bin worker's + CPU-side source), and the "sampled n of N" badge reports the overlay + actually drawn. - **T10 — the aggregate backdrop is continuous through transitions, and retires when the drill settles:** the density texture draws under the marks in every TRANSITIONAL drill state — entering, held, dying, exiting — @@ -440,8 +475,120 @@ invariants so future kinds don't regress them: trace, which drops the drill and with it the elision. Non-exact replies (anything but `reduction: "none"`, including replies that don't say) never arm it. - -Any new tiered kind must state how it satisfies T1–T12 in its chart-kind +- **T13 — full-point windows are padded, aligned, cached, and never + re-requested:** T12's elision is only as good as the window it can elide + against, so both ends widen it. *Kernel side*, a points-tier reply ships + the largest ALIGNED window around the view whose exact count still fits + the budget (`interaction._padded_drill_window`): bounds snap outward to a + power-of-two grid over the trace's extent, per dimension + (`lod.aligned_window`), from a ladder of span targets + (`DRILL_PAD_TARGETS`, coarsest first, pyramid-count-gated then + exact-verified), floored at the raw view window. Alignment makes + consecutive pans resolve to the SAME window — dedupable, cacheable — and + neighboring windows tile; the padded span is hard-capped per axis + (`DRILL_PAD_SPAN_CAP`, well under the 1/256 re-encode bound) so the §16 + offset encoding centered on the padded window can always be re-tightened + by a deeper zoom's re-encode request. Nonlinear-axis traces skip padding + (raw-space alignment mis-sizes log windows near zero) and keep the exact + view window. `visible` counts the SHIPPED window; `lod_blend` stays keyed + on the VIEW's own count — padding widens what ships, not what the user + sees. *Client side*, a points reply for a new window RETIRES the previous + exact drill into a bounded per-trace LRU (`g.drillCache`, + `LOD_POINT_CACHE_WINDOWS`) instead of overwriting its buffers, and so does + a drill that dies outside its window; any later view covered by a cached + window promotes it back (`lodPromoteCachedDrill` — alpha-continuous, brush + mask re-derived locally) with no wire message, so pan ping-pong and + zoom-out/zoom-in sequences render entirely from the GPU. Cached windows + obey the live drill's geometry-only memory discipline (T11): outgrown ⇒ + freed on the frame, no kernel reply required (§27). Because promoted + windows carry retired `drill_seq`s, the kernel keeps a bounded subset + history (`Trace.drill_history`, `DRILL_HISTORY_KEEP`) so picks against a + recent retired window still translate exactly; expired seqs drop the pick + (§16 exact-or-nothing), and data changes clear the history. Finally, an + identical request never rides the wire twice: a `density_view` within half + an output texel per edge of the trace's last sent request (gesture-end and + settle emit sub-pixel twins; a grid shifted below half a texel is the same + picture) is suppressed — already answered ⇒ nothing to refresh (the reply + is deterministic for unchanged data; rebuilds reset the memo), still in + flight ⇒ the trace keeps waiting on the ORIGINAL request's seq and that + reply is accepted per-trace instead of dying to the global seq race + (bounded by the same 1200ms window as the T8 hold, so a lost reply can't + suppress forever). + + The AGGREGATE tier's own traffic is governed by a stronger rule, adopted + from field feedback on the 100M drilldown (a HAR showed ~2.7 MB + full-screen grids re-shipped on every pan/zoom step at 200–450% zoom for + what was the same aggregate with marginally different blur): **the + aggregate never refines.** Whatever density texture already covers the + view stands — however blurry — until the view could plausibly resolve + into REAL points; only then is a round-trip worth anything, and the + kernel answers it with exact points once the window's count fits the + budget. Blur at intermediate zooms is an accepted, recorded cost. + - **The points-band gate** (`lodAggregateStands`): a `density_view` goes + out only when the estimated in-view count sits within + `LOD_DIRECT_POINT_BUDGET × LOD_POINTS_REQUEST_BAND`. Two independent + estimators, LOWER wins (an over-estimate must never hold a view in blur + when either signal says points could be close): the smallest cached + density window CONTAINING the view (recorded `visible`, exact for its + window, area-scaled — seeded by the home texture's first-payload + count), and the retained deterministic sample counted in-view and + re-weighted by `visible/n` (`lodSampleViewCount`) — a fixed-rate + thinning of the whole trace, so it follows the data's ACTUAL + distribution where area-scaling assumes uniformity and over-estimates + sparse tails by orders of magnitude (~65 expected sample rows right at + the band ⇒ ±12% noise; kernel-attached clients never DRAW the sample, + T9 — estimating from it CPU-side is what it is retained for). A trace + with no recorded counts anywhere always requests. A fresh grid for an + already-cached window supersedes its unpinned twin + (`lodRememberDensity` dedupe), so the gate always reads current facts. + *Display side — the stepped ladder:* the aggregate sharpens in + QUANTIZED steps, never per view. While standing, the one density + request allowed is the next LADDER STEP window + (`lodAggregateStepWindow`): the view snapped outward to a + power-of-`LOD_AGG_STEP_FACTOR` block grid over the data extent, per + axis, at most `LOD_AGG_STEP_MAX` steps below home, requested only when + every covering texture is coarser than the step + (`LOD_AGG_STEP_SLACK`). Quantization makes step windows pan-stable and + dedupable — every view in a region resolves to the SAME window, pans + inside a step repaint nothing, revisits hit the cache — so a zoom sees + at most `LOD_AGG_STEP_MAX` smooth-to-smooth swaps before points, and + worst-case softness is bounded (≈ `LOD_AGG_STEP_FACTOR`× stretch per + axis) instead of unbounded home-stretch. Per-view refinement was the + recorded failure mode twice over: multi-MB re-ships per pan/zoom step + (the HAR), and fresh textures per view reading as jumping. A step + reply is the ONE density reply that may repaint a covered view + (matched against the marked request window, `g._stepReqWin`); every + other covered reply — the band probes above all — lands as a + facts-only cache entry (window + exact count, + `lodRememberDensityFacts`) that recalibrates this gate: the band's + exact grids have a hard speckled character (sparse cells at the + points' own §2 alpha), and repainting the smooth surface with one per + probe read as jumping between zoom levels (field capture). A reply for + an UNCOVERED view still applies — silence must never blank a frame + (T1) — and standalone clients apply everything (their re-binned grids + are the only refinement they have). Recorded trade-off: inside the + points band the raw-view probe takes the single request slot, so a + view can briefly outrun its ladder step until points land. + - **Source-clamped grids:** the transition-band replies that do go out + stay cheap — a pyramid-served reply never composes more cells than the + finest level resolves under the window + (`interaction._pyramid_source_shape`, `ceil(base·frac)+1` per axis); a + full-screen grid of upsampled base cells is the same picture at several + times the bytes, and the client's texture filtering reproduces the + upscale. Exact/spatial grids (true full-detail bins) keep screen + resolution. + - **One texture per frame (recorded reversal):** a fine-over-broad detail + layer — drawing the smallest cached texture overlapping the view on top + of the broad backdrop during pans — was tried and REVERTED: density + textures alpha-composite, so the overlap double-counts opacity, and + each window's texture is baked against its own eased normMax, so the + seam is also a brightness step (field capture: a stale darker + rectangle). Doing it right requires the backdrop scissored out of the + detail region plus a shared normalization across cached textures; + under the never-refines rule the case barely arises, so the tier draws + one texture per frame. + +Any new tiered kind must state how it satisfies T1–T13 in its chart-kind contract entry before it lands. --- @@ -461,8 +608,9 @@ contract entry before it lands. (`_payload._density_trace_spec`), `density_view` (exact and pyramid paths), the static SVG/PNG exporters, and the standalone re-bin worker; DENSITY_FS gained the `u_meanColor` branch (premultiplied RGBA8 texture, - count tone-curve baked into alpha at upload so bilinear filtering weights - color by coverage); the drill handoff became intensity-only (§2 rule 5). + the §2 rule-1 physical alpha baked at upload so bilinear filtering + weights color by coverage); mean-color drills swap at native opacity + with no intensity handoff (§2 rule 5; count-only drills keep it). The colorbar for a continuous channel now stays on density traces — the surface really shows the channel's colors. 3. Legend/badge rendering from spec facts (`color_agg`, `dropped_channels`) @@ -477,14 +625,18 @@ contract entry before it lands. row-order independence, rare-category floors, validation, dtype-preserving saturation, and subset-monotonicity across levels and viewport narrowing. -5. **Done for exact density views:** hybrid scatter mode renders density plus - deterministic sampled exact points, with a visible "sampled n of N" badge. - Pyramid-served density views still need tile-aware sample overlays so the - same anti-shimmer contract holds without rescanning raw rows. Until they - exist, a sample-less reply's window carries no overlay of its own and the - T9 window pairing serves the best cached window's sample instead (the - home overlay above all, so zoom-outs keep representative points), bounded - by the coverage fade when nothing covers the view. +5. **Revised by #225 (resolvability gate, T9):** interactive density replies + no longer ship samples at all — a fixed-size sample above the drill budget + reads as individual data points at a zoom where real points are sub-pixel, + and the mean-color surface (§2) is the truthful stand-alone + representation. The retained first-payload sample remains (the standalone + re-bin worker's CPU source) and draws — with its "sampled n of N" badge — + only when the estimated in-view count fits the direct budget: kernel mode + ships real points before that, so the hybrid look is transient at most; + standalone exports surface it once a zoom resolves it; datasets under the + budget keep it everywhere. The deterministic-sampling utilities (item 4) + stay: the first-payload overlay and any future resolvable-tier subset are + built from them. **Phase 3 — pyramid (build + serve shipped; client cache and bench gate open)** 6. **Done (count + mean-color planes):** `src/tiles.rs` builds a square count diff --git a/spec/design/wire-protocol.md b/spec/design/wire-protocol.md index 57304179..0f94dde9 100644 --- a/spec/design/wire-protocol.md +++ b/spec/design/wire-protocol.md @@ -62,18 +62,33 @@ produces no traces, there is no reply at all (silence, not an empty message). `trace` id. `w` defaults to `512`, `h` to `384`; the client sends the rounded plot width and height. A trace that is not in density mode yields `{"traces": []}`, which is dropped rather than sent. Not every view change -produces a request: a view fully contained in a live drill's window is elided -client-side when that drill shipped its window exactly (`reduction: "none"` — -the subset already holds every point of any contained view; LOD doc §5 T12). -The elision ends, and one request goes out purely to re-center the §16 f32 -offset encoding, once the view span drops below 1/256 of the drilled window's -span on either axis. +produces a request: a view fully contained in an exact shipped window — the +live drill's, or a retired cached point window the client promotes back — is +elided client-side (`reduction: "none"` means the subset already holds every +point of any contained view; LOD doc §5 T12/T13). The elision ends, and one +request goes out purely to re-center the §16 f32 offset encoding, once the +view span drops below 1/256 of the drilled window's span on either axis. +A request within half an output texel per edge of the trace's last sent +request (same screen size) is also suppressed (T13): if that request was +already answered there is nothing to refresh (replies are deterministic for +unchanged data; data changes rebuild the GPU trace and reset the memo), and +if it is still in flight the trace keeps waiting on the original request's +`seq`, whose reply is then accepted per-trace instead of dying to the global +seq race. More fundamentally, the aggregate tier never refines (LOD doc T13, +revised): a `density_view` goes out only when the estimated in-view count — +the smallest cached window containing the view, area-scaled, seeded by the +first payload's recorded count — sits within the points band +(`LOD_DIRECT_POINT_BUDGET × LOD_POINTS_REQUEST_BAND`). Outside that band the +covering texture stands, however blurry; traces with no recorded counts +always request. **`pick`** — `trace` and `index` pass through `_integer_id`. `index` is a *shipped-vertex* index, translated kernel-side to a canonical row when the shipped copy dropped non-finite rows. `drill_seq`, when present, is the drill -subset version the client picked against; a mismatch resolves to `row: null` -rather than to a row in a dead index space. +subset version the client picked against; a non-current seq translates +through the kernel's bounded subset history when it is still remembered (the +client may be drawing a retired cached point window, LOD doc T13) and +resolves to `row: null` otherwise — never to a row in a dead index space. **`click`** — same fields and same `fig.pick` resolution as `pick`, minus `seq`; it fires `on_click` and returns nothing. @@ -135,9 +150,19 @@ states which representation this view resolved to: constant) instead and no color plane — the mean of a constant IS the constant, so the client tints the count texture. Count always rides `buf` as log-u8 and drives only the drawn **alpha**; renderers must never - colormap counts when either color source is present. Optional `sample` is - the retained point-sample overlay. `binning` is `"exact"` or - `"pyramid-L"`. `x_range`/`y_range` are raw data endpoints, but the + colormap counts when either color source is present. Interactive replies + ship **no `sample`** (#225): the only retained point-sample overlay is the + first-payload one, and the client draws it solely below the T9 + resolvable-count gate (a client still accepts a legacy reply-borne `sample` + and gates it the same way). `binning` is `"exact"` or + `"pyramid-L"`. Pyramid-served grids are **clamped to source + resolution** — never more cells per axis than the finest level resolves + under the window (a full-screen grid of upsampled cells is the same + picture at several times the bytes; the client's texture filtering does + the upscale). Exact and spatial grids are true full-detail bins at screen + resolution. `visible` — the window's point count — is the fact the + client's points-band gate scales to decide whether any later view is + worth a request at all (LOD doc T13). `x_range`/`y_range` are raw data endpoints, but the grid's cells are **uniform in the axis's scale coordinates** (identical to raw on a linear axis): on a log/symlog axis the kernel bins transformed values so every cell covers the same strip of screen, and renderers @@ -147,12 +172,16 @@ states which representation this view resolved to: - `mode: "points"` — the deep-zoom drill: `{id, mode, tier: "direct", visible, reduction: "none", x_range, y_range, x, y, color, size, density_val, lod_blend, drill_seq, style}`. - `x_range`/`y_range` are the window these points cover; the client falls - back to the density overview the moment the view leaves it. `density_val` - (per-point local log-density) and `lod_blend` drive the intensity-only - handoff: hue is continuous by construction because the aggregate surface - wears the mean point color, so no `density_colormap` field rides this - message (clients ignore one if present). + `x_range`/`y_range` are the window these points cover — usually a padded + ALIGNED superset of the requested view (LOD doc T13; the raw view window on + nonlinear axes or when no padding fits the budget) — and the client falls + back to the density overview the moment the view leaves it. `visible` + counts the points of the SHIPPED window; `lod_blend` stays keyed on the + requested view's own count. `density_val` (per-point local log-density) and + `lod_blend` drive the intensity-only handoff: hue is continuous by + construction because the aggregate surface wears the mean point color, so + no `density_colormap` field rides this message (clients ignore one if + present). Channel encodings on this (and the sampled-overlay) live wire: `x`/`y` are offset-encoded f32 (position precision is load-bearing, dossier §16), but diff --git a/tests/test_density_mean_color.py b/tests/test_density_mean_color.py index 75d813c3..84ec2add 100644 --- a/tests/test_density_mean_color.py +++ b/tests/test_density_mean_color.py @@ -1,9 +1,10 @@ """Mean-color density surface (LOD doc §2): the aggregated view wears the data's own colors — per-cell alpha-weighted mean of the resolved point -colors, averaged in linear light — while the binned count drives only the -alpha channel. These tests pin the kernel against a NumPy oracle (the LOD -doc's exit criterion), the wire shape across the initial emit / exact -density_view / pyramid paths, and the static exporters' color law. +colors, averaged in linear light — composited at the points' own alpha +(`1 − (1 − ā)^count`, the physical downsample). These tests pin the kernel +against a NumPy oracle (the LOD doc's exit criterion), the wire shape across +the initial emit / exact density_view / pyramid paths, and the static +exporters' color law. """ from __future__ import annotations @@ -256,6 +257,27 @@ def test_svg_export_density_uses_mean_colors(): assert (right[:, :3] == palette[1, :3]).all() +def test_physical_density_alpha_law(): + # LOD doc §2 rule 1: alpha = 1 - (1 - a_pt)^count with a_pt = channel + # alpha x style opacity folded inside the exponent; empty/invisible + # cells stay 0; a_pt = 1 saturates any occupied cell; no window max + # enters anywhere. This is the exporters' twin of the client upload law. + from xy._svg import _physical_density_alpha + + counts = np.asarray([0.0, 1.0, 3.0, 50.0, 2.0, 5.0]) + mean_a = np.asarray([255, 255, 255, 255, 0, 255], dtype=np.uint8) + out = _physical_density_alpha(counts, mean_a, 0.72) + expect = lambda k: round(255 * (1.0 - (1.0 - 0.72) ** k)) # noqa: E731 + assert out[0] == 0 # empty + assert abs(int(out[1]) - expect(1)) <= 1 # one point IS the point alpha + assert abs(int(out[2]) - expect(3)) <= 1 + assert out[3] >= 254 # saturates like overplotted marks + assert out[4] == 0 # all-invisible cell never invents coverage + # Style opacity 1 + full channel alpha: any occupied cell is opaque. + full = _physical_density_alpha(counts, np.full(6, 255, dtype=np.uint8), 1.0) + assert full[0] == 0 and (full[1:4] == 255).all() + + def test_resolve_bin_colors_modes(): # constant -> None (count-only grid + client tint) assert ( diff --git a/tests/test_density_physical_alpha.py b/tests/test_density_physical_alpha.py new file mode 100644 index 00000000..cba6a608 --- /dev/null +++ b/tests/test_density_physical_alpha.py @@ -0,0 +1,151 @@ +"""Mean-color density textures composite like the points they aggregate +(LOD doc §2 rule 1). + +A channel-bearing cell uploads at alpha `1 − (1 − ā)^count` — the physical +compositing of k overplotted points of mean straight alpha ā — with NO +window normalization: a one-point cell reads exactly like one point, dense +cells saturate exactly like overplotted marks, and lightness cannot swing +between windows or across the texture↔points boundary (the previous +per-window log-count tone curve did both; field capture on the 100M +drilldown). Count-only grids keep the log ramp — count is their only +structure — and mean-color drills swap at native opacity with no intensity +handoff (`density_val`/`lod_blend` ignored on such traces). + +This drives the real client in headless Chromium, intercepting the texture +upload to read the exact bytes. +""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pytest + +import xy +from conftest import run_browser_probe +from xy.export import find_chromium + +_RENDER_CALL = 'xy.renderStandalone(document.getElementById("chart"), spec, buf);' + +_PROBE = """ + const view = xy.renderStandalone(document.getElementById("chart"), spec, buf); + try { + view._drawNow(); + view._raf = null; + view._sampleRebinDisabled = true; + const g = view.gpuTraces.find((t) => t.tier === "density"); + const v0 = view.view0; + + // A mean-color reply: 4 cells with counts [0, 1, 3, 50], all wearing the + // same mean color at full channel alpha — the drawn per-point alpha is + // the trace's style opacity (0.72), folded INSIDE the exponent. + const gw = 4, gh = 1; + const counts = new Float32Array([0, 1, 3, 50]); + const abar = 0.72; // style opacity from the chart below + const rgba = new Uint8Array(gw * 4); + for (let i = 0; i < gw; i++) { + rgba[i * 4] = 200; rgba[i * 4 + 1] = 60; rgba[i * 4 + 2] = 30; + rgba[i * 4 + 3] = 255; + } + let uploaded = null; + const gl = view.gl; + const realTexImage = gl.texImage2D.bind(gl); + gl.texImage2D = function (...args) { + const data = args[args.length - 1]; + if (data && data.length === gw * gh * 4) uploaded = new Uint8Array(data); + return realTexImage(...args); + }; + view._onKernelMsg({ + type: "density_update", seq: view.seq, trace: g.trace.id, + traces: [{ + id: g.trace.id, mode: "density", visible: 5000000, + density: { + buf: 0, w: gw, h: gh, max: 50, + x_range: [v0.x0, v0.x1], y_range: [v0.y0, v0.y1], + rgba: 1, color_agg: "mean", + }, + }], + }, [counts.buffer, rgba.buffer]); + gl.texImage2D = realTexImage; + + const a = uploaded ? [uploaded[3], uploaded[7], uploaded[11], uploaded[15]] : null; + const expect = (k) => Math.round(255 * (1 - Math.pow(1 - abar, k))); + // Physical law, no window-max dependence: empty stays 0, k=1 IS the + // drawn per-point alpha, k=3 composites up, k=50 saturates past the + // style opacity exactly like overplotted marks. + const law = a && a[0] === 0 && + Math.abs(a[1] - expect(1)) <= 1 && + Math.abs(a[2] - expect(3)) <= 1 && + a[3] >= 254; + // No exposure easing on mean-color grids: nothing schedules re-uploads. + const noNormAnim = g._densityNormAnim === null || g._densityNormAnim === undefined; + + // A drill landing on this mean-color surface carries density_val and a + // blend weight — both must be ignored: marks enter at native opacity. + const N = 50; + const xs = new Float32Array(N), ys = new Float32Array(N), dv = new Float32Array(N); + for (let i = 0; i < N; i++) { + xs[i] = v0.x0 + (v0.x1 - v0.x0) * ((i * 0.618) % 1); + ys[i] = v0.y0 + (v0.y1 - v0.y0) * ((i * 0.314) % 1); + dv[i] = 0.9; + } + view._applyDrill(g, { + id: g.trace.id, mode: "points", visible: N, reduction: "none", drill_seq: 1, + x: { buf: 0, offset: 0, scale: 1, len: N }, + y: { buf: 1, offset: 0, scale: 1, len: N }, + x_range: [v0.x0, v0.x1], y_range: [v0.y0, v0.y1], + density_val: { buf: 2 }, lod_blend: 0.85, + }, [xs.buffer, ys.buffer, dv.buffer]); + const noHandoff = g.drill && g.drill.lodBlend === 0 && !g.drill.dBuf; + + document.body.setAttribute("data-xy-alpha-probe", JSON.stringify({ + hasDensity: !!g, gotUpload: !!uploaded, alphas: a, law, noNormAnim, noHandoff, + })); + } catch (err) { + document.body.setAttribute( + "data-xy-alpha-probe-error", + String((err && err.stack) || err), + ); + } +""" + + +def _density_html() -> str: + rng = np.random.default_rng(0) + n = 60_000 + x = rng.normal(0.0, 1.0, n) + y = rng.normal(0.0, 1.0, n) + c = np.hypot(x, y) + chart = xy.scatter_chart( + xy.scatter(x, y, color=c, colormap="viridis", opacity=0.72, density=True), + xy.x_axis(), + xy.y_axis(), + width=480, + height=360, + ) + html = chart.to_html() + assert _RENDER_CALL in html + return html + + +def test_mean_color_texture_uses_physical_alpha(tmp_path: Path) -> None: + chromium = find_chromium() + if chromium is None: + pytest.skip("Chromium unavailable") + + document = _density_html().replace(_RENDER_CALL, _PROBE) + result = run_browser_probe( + chromium, + document, + tmp_path / "density_physical_alpha.html", + "data-xy-alpha-probe", + label="mean-color physical-alpha probe", + ) + + assert result["hasDensity"] is True + assert result["gotUpload"] is True + # alpha(k) = 1-(1-0.72)^k, independent of the window max: 0, 184, 250, 255. + assert result["law"] is True, result["alphas"] + assert result["noNormAnim"] is True + assert result["noHandoff"] is True diff --git a/tests/test_density_request_economy.py b/tests/test_density_request_economy.py new file mode 100644 index 00000000..8d74594b --- /dev/null +++ b/tests/test_density_request_economy.py @@ -0,0 +1,258 @@ +"""The aggregate sharpens in QUANTIZED ladder steps and probes the points +band with raw views (LOD doc T13, revised on #225 field feedback). + +The field HAR behind this (100M live drilldown): every pan and zoom step +re-requested a multi-MB density grid — including back-to-back windows +differing by sub-pixel amounts — for a picture that was the same aggregate +with marginally different blur, and per-view textures read as zoom/pan +jumping. The revised contract: while the estimated in-view count sits above +the points band (LOD_POINTS_REQUEST_BAND × the direct budget), the only +density request allowed is the next LADDER STEP — the view snapped outward +to a power-of-LOD_AGG_STEP_FACTOR block grid over the extent, at most +LOD_AGG_STEP_MAX steps below home — whose reply is the one density reply +that may repaint a covered view (a smooth-to-smooth swap; pans inside a +step re-resolve to the same window). Inside the band the RAW view is +requested so the kernel decides the tier with real counts; its density +replies land as facts only. Sub-texel twins of the last sent request are +suppressed outright, and the frame always draws ONE aggregate texture (a +tried fine-over-broad detail layer was reverted: density textures +alpha-composite, so overlaps double-count opacity, and per-window +normalization makes the seam a brightness step — the field capture's +"stale rectangle"). + +This drives the real client in headless Chromium with a captured comm. +""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pytest + +import xy +from conftest import run_browser_probe +from xy.export import find_chromium + +_RENDER_CALL = 'xy.renderStandalone(document.getElementById("chart"), spec, buf);' + +_PROBE = """ + const view = xy.renderStandalone(document.getElementById("chart"), spec, buf); + try { + view._drawNow(); + view._raf = null; + view._sampleRebinDisabled = true; + const g = view.gpuTraces.find((t) => t.tier === "density"); + let clk = 100000; + view._now = () => clk; + g._densityNormAnim = null; + + const v0 = view.view0; + const cx = (v0.x0 + v0.x1) / 2, cy = (v0.y0 + v0.y1) / 2; + const sx = v0.x1 - v0.x0, sy = v0.y1 - v0.y0; + const sent = []; + view.comm = { send: (m) => sent.push(m) }; + const densitySent = () => sent.filter((m) => m.type === "density_view").length; + const request = (v) => { + const before = densitySent(); + view._scheduleViewRequest(view._viewFrom(v), { delay: 0 }); + return densitySent() - before; + }; + const viewAt = (fcx, fcy, fspanx, fspany) => ({ + x0: v0.x0 + sx * fcx - sx * fspanx / 2, x1: v0.x0 + sx * fcx + sx * fspanx / 2, + y0: v0.y0 + sy * fcy - sy * fspany / 2, y1: v0.y0 + sy * fcy + sy * fspany / 2, + }); + + // The home texture carries the payload's recorded count (60k here). + const homeHasCount = Number.isFinite(g.density.visible); + // Model a huge cloud: this probe chart holds 60k points (points-band + // everywhere, so nothing would ever stand). Re-weight the retained + // sample's claim so each sample row represents ~1.2M points — the + // distribution-true estimator then reads like a 10B-point trace, and + // the window-count fixtures below drive the area estimator. + g.sampleOverlay.sample.visible = 10000000000; + + // A density reply for window W = central 50% of home, 10M in-window. + const applyDensity = (x0d, x1d, y0d, y1d, gw, gh, visible) => { + const grid = new Float32Array(gw * gh).fill(3); + view._onKernelMsg({ + type: "density_update", seq: view.seq, trace: g.trace.id, + traces: [{ + id: g.trace.id, mode: "density", visible, + binning: "pyramid-L0", + density: { + buf: 0, w: gw, h: gh, max: 3, + x_range: [x0d, x1d], y_range: [y0d, y1d], + }, + }], + }, [grid.buffer]); + }; + const wx0 = cx - sx * 0.25, wx1 = cx + sx * 0.25; + const wy0 = cy - sy * 0.25, wy1 = cy + sy * 0.25; + const shownBefore = g.density; + applyDensity(wx0, wx1, wy0, wy1, 200, 150, 10000000); + // Display side of the stands-rule: the home texture covers the view, so + // the reply changed no pixels — it landed as a FACTS-ONLY cache entry + // (window + exact count) that the points-band gate reads below. The + // field capture behind this: transition-band exact grids repainting the + // smooth standing surface on every probe read as zoom-level jumping. + const standingHeld = g.density === shownBefore; + const factsStored = (g.densityCache || []).some( + (c) => c && !c.tex && c.visible === 10000000); + + // Zooming inside W at half its span: ~2.5M estimated in view — nowhere + // near points territory, so the aggregate stands. The one request the + // stepped ladder allows: the QUANTIZED step-1 window (the view snapped + // outward to extent/4 blocks — here exactly W's bounds), never the raw + // view window. + const stepView = viewAt(0.5, 0.5, 0.25, 0.25); + const stepSent = request(stepView) === 1; + const stepMsg = sent.filter((m) => m.type === "density_view").pop(); + const stepTol = sx * 1e-9; + const stepAligned = stepMsg && + Math.abs(stepMsg.x0 - wx0) <= stepTol && Math.abs(stepMsg.x1 - wx1) <= stepTol && + Math.abs(stepMsg.y0 - wy0) <= stepTol && Math.abs(stepMsg.y1 - wy1) <= stepTol; + const stepMarked = Array.isArray(g._stepReqWin); + // The step reply is the ONE density reply allowed to repaint a covered + // view: it replaces the standing texture (smooth-to-smooth swap). + applyDensity(stepMsg.x0, stepMsg.x1, stepMsg.y0, stepMsg.y1, 200, 150, 10000000); + const stepApplied = g.density !== shownBefore && + Math.abs(g.density.xRange[0] - wx0) <= stepTol && g._stepReqWin === null; + + // Re-requesting a view the step texture now covers sends nothing. + const insideElided = request(stepView) === 0; + const elisionClearedPending = g._lodPendingView === null; + + // Deeper inside W (1/16 of its area): ~625k estimated — inside the + // points band (budget x 4), so the RAW VIEW request goes out and the + // kernel decides with real counts. + const bandView = viewAt(0.5, 0.5, 0.12, 0.12); + const bandRequested = request(bandView) === 1; + const bandMsg = sent.filter((m) => m.type === "density_view").pop(); + const bandIsRawView = bandMsg && + Math.abs(bandMsg.x0 - bandView.x0) <= stepTol && + Math.abs(bandMsg.x1 - bandView.x1) <= stepTol; + + // A window whose count is already points-scale requests immediately. + const w2x0 = v0.x0 + sx * 0.10, w2x1 = v0.x0 + sx * 0.40; + const w2y0 = v0.y0 + sy * 0.10, w2y1 = v0.y0 + sy * 0.40; + applyDensity(w2x0, w2x1, w2y0, w2y1, 100, 75, 300000); + const nearDrillRequested = request(viewAt(0.25, 0.25, 0.15, 0.15)) === 1; + + // Sub-texel dedup: a settle re-request shifted by 0.2 output texels is + // the same picture — suppressed while the first is in flight; a shift of + // 3 texels is a genuinely different window — sent. Restore the honest + // 60k sample weighting first so these views sit in the points band and + // request RAW windows (a standing view would quantize to the same + // aligned step window under any sub-block shift, gating on alignment + // rather than the dedup memo this section pins). + g.sampleOverlay.sample.visible = 60000; + const plotW = Math.round(view.plot.w); + const base = viewAt(0.75, 0.75, 0.2, 0.2); + const sentBase = request(base) === 1; + const texel = (base.x1 - base.x0) / plotW; + const nudged = { ...base, x0: base.x0 + texel * 0.2, x1: base.x1 + texel * 0.2 }; + const subTexelSuppressed = request(nudged) === 0; + const shifted = { ...base, x0: base.x0 + texel * 3, x1: base.x1 + texel * 3 }; + const texelsRequested = request(shifted) === 1; + + // One aggregate texture per frame (the reverted detail layer must stay + // reverted): straddle W's edge — home contains the view and draws alone + // once the crossfade settles; W's finer texture must NOT stack on top. + view.view = viewAt(0.75, 0.5, 0.5, 0.5); + view._drawNow(); + clk += 1000; // land the density crossfade + view._drawNow(); + let densityDraws = 0; + const real = view._drawDensity; + view._drawDensity = function (gg, dd, op) { + if (gg === g) densityDraws++; + return real.apply(this, arguments); + }; + view._drawNow(); + view._drawDensity = real; + + // Coverage failure still applies the reply: a view panned past every + // cached window must never sit on a blank frame waiting for points (T1). + view.view = { x0: v0.x1 + sx, x1: v0.x1 + 2 * sx, y0: v0.y0, y1: v0.y1 }; + applyDensity(v0.x1 + sx, v0.x1 + 2 * sx, v0.y0, v0.y1, 50, 40, 5000000); + const uncoveredApplied = g.density !== shownBefore && g.density.visible === 5000000; + + document.body.setAttribute("data-xy-economy-probe", JSON.stringify({ + hasDensity: !!g, homeHasCount, standingHeld, factsStored, + stepSent, stepAligned, stepMarked, stepApplied, + insideElided, elisionClearedPending, bandRequested, bandIsRawView, + nearDrillRequested, + sentBase, subTexelSuppressed, texelsRequested, + densityDraws, uncoveredApplied, + })); + } catch (err) { + document.body.setAttribute( + "data-xy-economy-probe-error", + String((err && err.stack) || err), + ); + } +""" + + +def _density_html() -> str: + rng = np.random.default_rng(0) + n = 60_000 + x = rng.normal(0.0, 1.0, n) + y = rng.normal(0.0, 1.0, n) + chart = xy.scatter_chart( + xy.scatter(x, y, density=True), + xy.x_axis(), + xy.y_axis(), + width=480, + height=360, + ) + html = chart.to_html() + assert _RENDER_CALL in html + return html + + +def test_aggregate_stands_until_points_band(tmp_path: Path) -> None: + chromium = find_chromium() + if chromium is None: + pytest.skip("Chromium unavailable") + + document = _density_html().replace(_RENDER_CALL, _PROBE) + result = run_browser_probe( + chromium, + document, + tmp_path / "density_request_economy.html", + "data-xy-economy-probe", + label="density request-economy probe", + ) + + assert result["hasDensity"] is True + assert result["homeHasCount"] is True + # Display side: a mid-band aggregate reply repaints NOTHING while a + # covering texture stands — it lands as facts for the gate; a reply for + # an uncovered view still applies (never a blank frame, T1). + assert result["standingHeld"] is True + assert result["factsStored"] is True + assert result["uncoveredApplied"] is True + # The stepped ladder: a standing view whose covering texture is coarser + # than its step requests the QUANTIZED aligned step window (never the + # raw view window), marks it, and that reply repaints; the same view + # afterwards requests nothing. + assert result["stepSent"] is True + assert result["stepAligned"] is True + assert result["stepMarked"] is True + assert result["stepApplied"] is True + assert result["insideElided"] is True + assert result["elisionClearedPending"] is True + # The points band and points-scale windows still request the RAW view — + # the kernel keeps deciding with real counts everywhere it could answer + # sharper. + assert result["bandRequested"] is True + assert result["bandIsRawView"] is True + assert result["nearDrillRequested"] is True + # Sub-texel dedup: same picture, no second request; a real shift sends. + assert result["sentBase"] is True + assert result["subTexelSuppressed"] is True + assert result["texelsRequested"] is True + # Exactly one aggregate texture per settled frame — no detail stacking. + assert result["densityDraws"] == 1 diff --git a/tests/test_density_sample_resolvability_gate.py b/tests/test_density_sample_resolvability_gate.py new file mode 100644 index 00000000..14dbc2fc --- /dev/null +++ b/tests/test_density_sample_resolvability_gate.py @@ -0,0 +1,218 @@ +"""Sample overlays draw only below the resolvable-count gate (#225). + +Above the drill budget, a fixed-size sample reads as individual data points at +a zoom where real points are sub-pixel — sampling above the resolution of the +graph misrepresents the dataset (the issue's field capture: zooming out from a +drilled window brought "individual points" back over a 100M-point cloud). The +client therefore gates every overlay candidate on the estimated in-view count: +the overlay's recorded window count scaled by the view's share of its window, +drawn only when that fits LOD_DIRECT_POINT_BUDGET. In kernel mode the gate +makes the hybrid look transient at most (interactive density replies ship no +samples anymore; real points arrive when a window fits the budget); standalone +exports keep the overlay as their only point representation once a zoom +resolves it. + +This drives the real client in headless Chromium: an overlay whose window +holds far more points than the budget must NOT draw even when its window +covers the view exactly (badge off — density-only), while zooming deep enough +into the same window that the estimated in-view count fits the budget brings +the overlay (and badge) back. With a kernel attached the overlay never draws +at all — resolvable views get REAL points from the kernel, so retained +sample rows there read as data that isn't (#225 field follow-up). +""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pytest + +import xy +from conftest import run_browser_probe +from xy.export import find_chromium + +_RENDER_CALL = 'xy.renderStandalone(document.getElementById("chart"), spec, buf);' + +_PROBE = """ + const view = xy.renderStandalone(document.getElementById("chart"), spec, buf); + try { + view._drawNow(); + view._raf = null; + view._sampleRebinDisabled = true; + const g = view.gpuTraces.find((t) => t.tier === "density"); + let clk = 100000; + view._now = () => clk; + g._densityNormAnim = null; + + const v0 = view.view0; + const cx = (v0.x0 + v0.x1) / 2, cy = (v0.y0 + v0.y1) / 2; + const sx = v0.x1 - v0.x0, sy = v0.y1 - v0.y0; + // Window W = central 20% of home; its sample CLAIMS 10M points in-window + // (a 100M-point-cloud zoom level) — far past the 200k direct budget. + const wx0 = cx - sx * 0.1, wx1 = cx + sx * 0.1; + const wy0 = cy - sy * 0.1, wy1 = cy + sy * 0.1; + const N = 400; + const xs = new Float32Array(N), ys = new Float32Array(N); + for (let i = 0; i < N; i++) { + xs[i] = wx0 + (wx1 - wx0) * ((i * 0.6180339887) % 1); + ys[i] = wy0 + (wy1 - wy0) * ((i * 0.3141592653) % 1); + } + const gw = 4, gh = 4; + const grid = new Float32Array(gw * gh).fill(3); + view._onKernelMsg({ + type: "density_update", seq: view.seq, trace: g.trace.id, + traces: [{ + id: g.trace.id, mode: "density", visible: 10000000, + density: { + buf: 0, w: gw, h: gh, max: 3, + x_range: [wx0, wx1], y_range: [wy0, wy1], + sample: { + n: N, visible: 10000000, + x: { buf: 1, offset: 0, scale: 1, len: N }, + y: { buf: 2, offset: 0, scale: 1, len: N }, + x_range: [wx0, wx1], y_range: [wy0, wy1], + }, + }, + }], + }, [grid.buffer, xs.buffer, ys.buffer]); + + const sampleDrawn = () => { + let drawn = null; + const real = view._drawPoints; + view._drawPoints = function (d, xm, ym, alpha) { + if (d && d.tier === "sampled") drawn = { n: d.n, alpha: alpha === undefined ? 1 : alpha }; + return real.apply(this, arguments); + }; + view._drawNow(); + view._drawPoints = real; + return drawn; + }; + const badgeShowsSample = () => + view._reductionBadgeItems().some((i) => i.indexOf("sampled") === 0); + const zoomTo = (f) => { + view.view = { + x0: cx - (wx1 - wx0) * f / 2, x1: cx + (wx1 - wx0) * f / 2, + y0: cy - (wy1 - wy0) * f / 2, y1: cy + (wy1 - wy0) * f / 2, + }; + }; + + // Model the 100M-cloud scenario: the HOME overlay too must claim a count + // far past the budget (the real chart under this probe holds only 60k + // points, whose overlay would honestly — and correctly — keep drawing). + const homeOverlay = g.sampleOverlay; + const homeVisible = homeOverlay.sample.visible; + homeOverlay.sample.visible = 100000000; + + // At W itself: 10M estimated in view — W's overlay must NOT draw even + // though its window covers the view exactly, and the home overlay (1e8) + // is gated too. Density-only, badge off. + zoomTo(1); + const atWindow = sampleDrawn(); + const atWindowBadge = badgeShowsSample(); + + // At home: nothing resolvable anywhere — the #225 field capture's fix: + // no individual points over a 100M-point aggregate. + view.view = { ...v0 }; + const atBigHome = sampleDrawn(); + + // 1/16 of W linearly (1/256 of the area): W's estimated in-view count is + // ~39k <= budget — the overlay (and badge) return: points are resolvable. + zoomTo(1 / 16); + const deepIn = sampleDrawn(); + const deepInBadge = badgeShowsSample(); + + // Just under the gate boundary from above: 1/4 linear = 1/16 area → + // 625k estimated, still over budget — still density-only. + zoomTo(1 / 4); + const midway = sampleDrawn(); + + // Restore the honest 60k home count: a dataset under the budget is + // resolvable everywhere and keeps the hybrid look, home included. + homeOverlay.sample.visible = homeVisible; + view.view = { ...v0 }; + const atHome = sampleDrawn(); + const atHomeBadge = badgeShowsSample(); + + // Kernel-attached clients NEVER draw retained samples (#225 field + // follow-up): wherever a view is resolvable, the kernel ships REAL + // points — a handful of arbitrary sample rows at full alpha there reads + // as data that isn't. The overlay is the standalone client's fallback. + view.comm = { send: () => {} }; + const kernelAtHome = sampleDrawn(); + zoomTo(1 / 16); + const kernelDeepIn = sampleDrawn(); + const kernelBadge = badgeShowsSample(); + view.comm = null; + view.view = { ...v0 }; + + document.body.setAttribute("data-xy-gate-probe", JSON.stringify({ + hasOverlays: !!(g.densityCache && g.densityCache.some((d) => d && d.overlay)), + windowN: N, + atWindow, atWindowBadge, atBigHome, deepIn, deepInBadge, midway, + atHome, atHomeBadge, kernelAtHome, kernelDeepIn, kernelBadge, + })); + } catch (err) { + document.body.setAttribute( + "data-xy-gate-probe-error", + String((err && err.stack) || err), + ); + } +""" + + +def _density_html() -> str: + rng = np.random.default_rng(0) + n = 60_000 + x = rng.normal(0.0, 1.0, n) + y = rng.normal(0.0, 1.0, n) + chart = xy.scatter_chart( + xy.scatter(x, y, density=True), + xy.x_axis(), + xy.y_axis(), + width=480, + height=360, + ) + html = chart.to_html() + assert _RENDER_CALL in html + return html + + +def test_sample_overlay_gated_by_resolvable_count(tmp_path: Path) -> None: + chromium = find_chromium() + if chromium is None: + pytest.skip("Chromium unavailable") + + document = _density_html().replace(_RENDER_CALL, _PROBE) + result = run_browser_probe( + chromium, + document, + tmp_path / "sample_resolvability_gate.html", + "data-xy-gate-probe", + label="sample resolvability-gate probe", + ) + + assert result["hasOverlays"] is True + # 10M points estimated in view: no overlay, no badge — density-only. This + # is the #225 contract: no sampled points above the graph's resolution. + assert result["atWindow"] is None + assert result["atWindowBadge"] is False + # Home over a (claimed) 100M cloud: density-only there too. + assert result["atBigHome"] is None + # Still over budget at 1/16 of the window's area: still density-only. + assert result["midway"] is None + # Deep enough that the estimated in-view count fits the budget: the + # overlay returns at full alpha, badge on. + assert result["deepIn"] is not None + assert result["deepIn"]["n"] == result["windowN"] + assert result["deepIn"]["alpha"] == 1 + assert result["deepInBadge"] is True + # A dataset under the budget is resolvable everywhere: its home overlay + # keeps drawing (nothing changes for small charts) — in STANDALONE mode. + assert result["atHome"] is not None + assert result["atHomeBadge"] is True + # With a kernel attached the overlay never draws at any zoom: resolvable + # views get real points from the kernel, not retained sample rows. + assert result["kernelAtHome"] is None + assert result["kernelDeepIn"] is None + assert result["kernelBadge"] is False diff --git a/tests/test_density_wire_economy.py b/tests/test_density_wire_economy.py new file mode 100644 index 00000000..5b2011b4 --- /dev/null +++ b/tests/test_density_wire_economy.py @@ -0,0 +1,105 @@ +"""Pyramid-served density replies never ship more grid than their source +resolves (§29 wire economy; LOD doc T13; #225 follow-up). + +The field HAR behind this: a 100M-point drilldown at 200-450% zoom shipped a +~2.7 MB full-screen grid per pan/zoom step — count plane plus RGBA8 mean-color +plane at one cell per screen pixel — while the pyramid's finest level only +resolves a few hundred cells under such a window. Composing past the source +just upsamples blocky cells the client's own texture filtering reproduces +from the source-resolution grid at a fraction of the bytes. The reply also +records `min_cell`, the finest attainable per-axis cell size, so the client +can elide requests a cached texture already answers. +""" + +from __future__ import annotations + +import math + +import numpy as np + +from xy import interaction +from xy._figure import Figure +from xy.config import PYRAMID_BASE_DIM + + +def test_pyramid_source_shape_math() -> None: + class _Col: + def __init__(self, lo, hi): + self.min, self.max = lo, hi + + class _T: + _pyr_base_dim = 1024 + + def __init__(self): + self.x = _Col(0.0, 100.0) + self.y = _Col(0.0, 50.0) + + t = _T() + # A quarter-extent window resolves ~256 (+1 straddle) source cells. + cells_x, cells_y = interaction._pyramid_source_shape(t, 10.0, 35.0, 5.0, 17.5) + assert cells_x == math.ceil(1024 * 0.25) + 1 + assert cells_y == math.ceil(1024 * 0.25) + 1 + # Windows wider than the extent clamp to the full base. + cells_x, _ = interaction._pyramid_source_shape(t, -50.0, 250.0, 0.0, 50.0) + assert cells_x == 1024 + 1 + # Degenerate extents refuse rather than divide by zero. + t.y = _Col(5.0, 5.0) + assert interaction._pyramid_source_shape(t, 0.0, 1.0, 0.0, 1.0) is None + + +def test_pyramid_reply_grid_bounded_by_source_cells() -> None: + n = 2_500_000 + rng = np.random.default_rng(3) + x = rng.uniform(0.0, 100.0, n) + y = rng.uniform(0.0, 100.0, n) + fig = Figure().scatter(x, y, density=True) + t = fig.traces[0] + + # Half-extent window: comfortably pyramid territory (~625k in-window). + upd, _ = fig.density_view(0, 25.0, 75.0, 25.0, 75.0, 512, 384) + tr = upd["traces"][0] + assert tr["mode"] == "density" + assert tr["binning"].startswith("pyramid") + d = tr["density"] + base = getattr(t, "_pyr_base_dim", 0) or PYRAMID_BASE_DIM + # The grid never exceeds the source cells under the window (+1 straddle). + assert d["w"] <= math.ceil(base * 0.5) + 1 + 1 + assert d["h"] <= math.ceil(base * 0.5) + 1 + 1 + # The reply carries the window's count — the fact the client's + # points-band gate (lodAggregateStands, T13) scales for later zooms. + assert tr["visible"] > 0 + + # A window near the drill budget (~260k in-window: over the budget, under + # the pyramid-serve margin) takes the exact path: true full-detail bins. + upd2, _ = fig.density_view(0, 0.0, 35.0, 0.0, 30.0, 512, 384) + tr2 = upd2["traces"][0] + assert tr2["mode"] == "density" + assert tr2["binning"] == "exact" + + +def test_pyramid_grid_clamped_to_source_resolution() -> None: + # Large enough that the per-cell target no longer coarsens the grid below + # the screen, so the source clamp is the binding constraint on x — the + # regime the 100M field HAR exercised. + n = 20_000_000 + rng = np.random.default_rng(7) + x = rng.uniform(0.0, 100.0, n) + y = rng.uniform(0.0, 100.0, n) + fig = Figure().scatter(x, y, density=True) + t = fig.traces[0] + + frac = 0.6 + upd, bufs = fig.density_view(0, 20.0, 20.0 + 100.0 * frac, 20.0, 80.0, 2000, 348) + tr = upd["traces"][0] + assert tr["mode"] == "density" + assert tr["binning"].startswith("pyramid") + d = tr["density"] + base = getattr(t, "_pyr_base_dim", 0) or PYRAMID_BASE_DIM + source_x = math.ceil(base * frac) + 1 + # x is clamped to source cells; y stays screen/plan-bounded — the grid's + # aspect visibly departs from the requested screen aspect. + assert d["w"] <= source_x + 1 + assert d["h"] >= 200 + assert d["w"] / d["h"] < (2000 / 348) * 0.9 + # One byte per cell on the count plane (§29 log-u8 wire). + assert len(bufs[d["buf"]]) == d["w"] * d["h"] diff --git a/tests/test_drill_point_window_cache.py b/tests/test_drill_point_window_cache.py new file mode 100644 index 00000000..5e391505 --- /dev/null +++ b/tests/test_drill_point_window_cache.py @@ -0,0 +1,216 @@ +"""Retired exact point windows are cached and promoted, never re-requested +(LOD doc T13). + +"Once we get points, we can render anything inside there without further +requests": a points reply for a NEW window retires the previous exact window +into a per-trace LRU cache instead of overwriting its buffers; any later view +covered by a cached window promotes it back to the live drill with no kernel +round-trip — pan ping-pong across a window boundary and zoom-out/zoom-in +sequences render entirely from the GPU. A drill dying OUTSIDE its window +(zoom-out to density) retires the same way, so diving back into the region it +covered is also free. Cached windows obey the live drill's geometry-only +memory discipline (T11): once the view outgrows a window past the drill +budget, its buffers free with no kernel reply required (§27). + +This drives the real client in headless Chromium with a captured comm. +""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pytest + +import xy +from conftest import run_browser_probe +from xy.export import find_chromium + +_RENDER_CALL = 'xy.renderStandalone(document.getElementById("chart"), spec, buf);' + +_PROBE = """ + const view = xy.renderStandalone(document.getElementById("chart"), spec, buf); + try { + view._drawNow(); + view._raf = null; + view._sampleRebinDisabled = true; + const g = view.gpuTraces.find((t) => t.tier === "density"); + let clk = 100000; + view._now = () => clk; + g._densityNormAnim = null; + + const v0 = view.view0; + const sx = v0.x1 - v0.x0, sy = v0.y1 - v0.y0; + const sent = []; + view.comm = { send: (m) => sent.push(m) }; + const densitySent = () => sent.filter((m) => m.type === "density_view").length; + + // A points reply for a window spanning [fx0, fx1] x [fy0, fy1] of home + // (fractions), claiming `visible` in-window points. + const N = 600; + let nextSeq = 1; + const applyPoints = (fx0, fx1, fy0, fy1, visible) => { + const wx0 = v0.x0 + sx * fx0, wx1 = v0.x0 + sx * fx1; + const wy0 = v0.y0 + sy * fy0, wy1 = v0.y0 + sy * fy1; + const xs = new Float32Array(N), ys = new Float32Array(N); + for (let i = 0; i < N; i++) { + xs[i] = wx0 + (wx1 - wx0) * ((i * 0.6180339887) % 1); + ys[i] = wy0 + (wy1 - wy0) * ((i * 0.3141592653) % 1); + } + view._applyDrill(g, { + id: g.trace.id, mode: "points", visible, reduction: "none", + drill_seq: nextSeq++, + x: { buf: 0, offset: 0, scale: 1, len: N }, + y: { buf: 1, offset: 0, scale: 1, len: N }, + x_range: [wx0, wx1], y_range: [wy0, wy1], + }, [xs.buffer, ys.buffer]); + return { x0: wx0, x1: wx1, y0: wy0, y1: wy1 }; + }; + const viewIn = (w, m) => ({ + x0: w.x0 + (w.x1 - w.x0) * m, x1: w.x1 - (w.x1 - w.x0) * m, + y0: w.y0 + (w.y1 - w.y0) * m, y1: w.y1 - (w.y1 - w.y0) * m, + }); + const sameWin = (a, b) => a && b && + Math.abs(a.x0 - b.x0) + Math.abs(a.x1 - b.x1) + + Math.abs(a.y0 - b.y0) + Math.abs(a.y1 - b.y1) < (sx + sy) * 1e-9; + const request = (v) => { + const before = densitySent(); + view._scheduleViewRequest(view._viewFrom(v), { delay: 0 }); + return densitySent() - before; + }; + + // Drill into window A, then a points reply for a DISJOINT window B lands + // (a pan): A retires into the cache instead of being overwritten. + const winA = applyPoints(0.10, 0.20, 0.10, 0.20, 600); + const seqA = 1; + const winB = applyPoints(0.30, 0.40, 0.30, 0.40, 600); + const retiredA = !!(g.drillCache && g.drillCache.length === 1 && + sameWin(g.drillCache[0].win, winA)); + const liveIsB = sameWin(g.drill.win, winB) && g.drill.seq === 2; + + // Pan back to a view inside A: promoted from the cache, no wire request, + // and picks keep speaking A's subset version (the kernel's history + // resolves it exactly). + const promoteElided = request(viewIn(winA, 0.25)) === 0; + const promotedIsA = sameWin(g.drill.win, winA) && g.drill.seq === seqA; + const swappedBToCache = !!(g.drillCache && g.drillCache.length === 1 && + sameWin(g.drillCache[0].win, winB)); + + // A view no cached window covers still requests. + const elsewhereRequested = request({ + x0: v0.x0 + sx * 0.6, x1: v0.x0 + sx * 0.7, + y0: v0.y0 + sy * 0.6, y1: v0.y0 + sy * 0.7 }) === 1; + + // LRU bound: windows C, D, E push the cache past its cap; the oldest + // retired windows fall off, newest survive. + applyPoints(0.50, 0.60, 0.50, 0.60, 600); // C (A -> cache) + applyPoints(0.62, 0.72, 0.62, 0.72, 600); // D (C -> cache) + applyPoints(0.74, 0.84, 0.74, 0.84, 600); // E (D -> cache) + const cacheBounded = g.drillCache.length <= 3; + + // Zoom-out to density: the drill dies; once its exit fade completes + // OUTSIDE its window, it retires into the cache (still exact for its + // window) instead of freeing. + view.view = { ...v0 }; + const gw = 4, gh = 4; + const grid = new Float32Array(gw * gh).fill(3); + view._onKernelMsg({ + type: "density_update", seq: view.seq, trace: g.trace.id, + traces: [{ + id: g.trace.id, mode: "density", visible: 60000, + density: { buf: 0, w: gw, h: gh, max: 3, + x_range: [v0.x0, v0.x1], y_range: [v0.y0, v0.y1] }, + }], + }, [grid.buffer]); + const dyingAfterDensity = g._drillDying === true; + view._drawNow(); // exit fade starts + clk += 500; // past LOD_EXIT_FADE_MS + view._drawNow(); // fade complete -> retire + const winE = { x0: v0.x0 + sx * 0.74, x1: v0.x0 + sx * 0.84, + y0: v0.y0 + sy * 0.74, y1: v0.y0 + sy * 0.84 }; + const retiredOnDeath = !g.drill && + g.drillCache.some((e) => sameWin(e.win, winE)); + + // Dive back into E's region: promoted, zero requests — the #225 workflow + // (drill, zoom out, drill the same spot) with no re-shipping. + const reviveElided = request(viewIn(winE, 0.25)) === 0; + const revivedIsE = !!(g.drill && sameWin(g.drill.win, winE)); + + // Geometry-only retirement for cached windows (T11 via T13): claim a + // budget-scale count on every cached window, zoom far out, and the sweep + // frees them on that frame — no kernel reply needed. + for (const e of g.drillCache) e.visible = 150000; + if (g.drill) g.drill.visible = 150000; + view.view = { ...v0 }; + view._drawNow(); + const sweptWhenOutgrown = !g.drillCache || g.drillCache.length === 0; + + document.body.setAttribute("data-xy-cache-probe", JSON.stringify({ + hasDensity: !!g, + retiredA, liveIsB, promoteElided, promotedIsA, swappedBToCache, + elsewhereRequested, cacheBounded, dyingAfterDensity, retiredOnDeath, + reviveElided, revivedIsE, sweptWhenOutgrown, + })); + } catch (err) { + document.body.setAttribute( + "data-xy-cache-probe-error", + String((err && err.stack) || err), + ); + } +""" + + +def _density_html() -> str: + rng = np.random.default_rng(0) + n = 60_000 + x = rng.normal(0.0, 1.0, n) + y = rng.normal(0.0, 1.0, n) + chart = xy.scatter_chart( + xy.scatter(x, y, density=True), + xy.x_axis(), + xy.y_axis(), + width=480, + height=360, + ) + html = chart.to_html() + assert _RENDER_CALL in html + return html + + +def test_point_windows_cache_promote_and_free(tmp_path: Path) -> None: + chromium = find_chromium() + if chromium is None: + pytest.skip("Chromium unavailable") + + document = _density_html().replace(_RENDER_CALL, _PROBE) + result = run_browser_probe( + chromium, + document, + tmp_path / "drill_point_window_cache.html", + "data-xy-cache-probe", + label="point-window cache probe", + ) + + assert result["hasDensity"] is True + # A new window's reply retires the old exact window instead of + # overwriting its buffers. + assert result["retiredA"] is True + assert result["liveIsB"] is True + # Pan back inside A: promoted from the cache with no wire request, and + # the promoted drill keeps A's subset version for picks. + assert result["promoteElided"] is True + assert result["promotedIsA"] is True + assert result["swappedBToCache"] is True + # Views nothing covers still go to the kernel. + assert result["elsewhereRequested"] is True + # The cache is bounded (LRU). + assert result["cacheBounded"] is True + # Zoom-out to density retires the dying drill (died outside its window) + # into the cache once the exit fade completes... + assert result["dyingAfterDensity"] is True + assert result["retiredOnDeath"] is True + # ...so diving back into that region is answered locally, zero requests. + assert result["reviveElided"] is True + assert result["revivedIsE"] is True + # Outgrown cached windows free on the frame, no kernel reply required. + assert result["sweptWhenOutgrown"] is True diff --git a/tests/test_drill_window_padding.py b/tests/test_drill_window_padding.py new file mode 100644 index 00000000..25b4522e --- /dev/null +++ b/tests/test_drill_window_padding.py @@ -0,0 +1,158 @@ +"""Padded aligned drill windows + drill-subset history (LOD doc T13, #225). + +A points-tier reply ships the largest ALIGNED window around the view whose +exact count still fits the budget: bounds snap outward to a power-of-two grid +over the trace's extent (`lod.aligned_window`), so consecutive pans resolve to +the SAME window and the client's point-window cache can key, dedupe, and +reuse full-point buffers by dimension. The raw view window is the floor — +never a subset, never over budget — and the §16 offset encoding re-centers on +the shipped window, so its span is hard-capped relative to the view +(DRILL_PAD_SPAN_CAP) to keep deep zooms inside the client's re-encode ladder. +""" + +from __future__ import annotations + +import numpy as np +import pytest + +from xy import lod +from xy._figure import Figure +from xy.config import DRILL_PAD_SPAN_CAP, SCATTER_DENSITY_THRESHOLD + + +def _uniform_fig(n: int, seed: int = 3) -> tuple[Figure, np.ndarray, np.ndarray]: + rng = np.random.default_rng(seed) + x = rng.uniform(0.0, 100.0, n) + y = rng.uniform(0.0, 100.0, n) + return Figure().scatter(x, y, density=True), x, y + + +def test_aligned_window_contains_input_and_is_pan_stable() -> None: + # Containment is the contract (client request elision needs it), even for + # windows panned past the data extent where nothing lives. + lo, hi = lod.aligned_window(-3.0, 5.0, 0.0, 100.0, 2.0) + assert lo <= -3.0 and hi >= 5.0 + # Same span bucket, different pan positions inside one block: identical + # aligned bounds — this is what makes cached windows reusable/dedupable. + a = lod.aligned_window(10.0, 14.0, 0.0, 128.0, 2.0) + b = lod.aligned_window(10.5, 14.5, 0.0, 128.0, 2.0) + assert a == b + # Bounds land on the extent's power-of-two grid. + span = 14.0 - 10.0 + level = int(np.ceil(np.log2(128.0 / (2.0 * span)))) + block = 128.0 / (1 << level) + assert a[0] / block == pytest.approx(round(a[0] / block)) + assert a[1] / block == pytest.approx(round(a[1] / block)) + + +def test_aligned_window_degenerate_inputs_pass_through() -> None: + assert lod.aligned_window(3.0, 3.0, 0.0, 10.0, 2.0) == (3.0, 3.0) # zero span + assert lod.aligned_window(1.0, 2.0, 5.0, 5.0, 2.0) == (1.0, 2.0) # zero extent + assert lod.aligned_window(1.0, 2.0, 0.0, np.inf, 2.0) == (1.0, 2.0) + # pad*span >= extent: the whole extent (unioned with the window so a view + # sticking past the data stays contained). + assert lod.aligned_window(1.0, 9.0, 0.0, 10.0, 4.0) == (0.0, 10.0) + assert lod.aligned_window(-2.0, 9.0, 0.0, 10.0, 4.0) == (-2.0, 10.0) + + +def test_drill_ships_padded_aligned_window_within_budget() -> None: + fig, x, y = _uniform_fig(SCATTER_DENSITY_THRESHOLD + 300_000) + upd, _ = fig.density_view(0, 40.0, 44.0, 40.0, 44.0, 512, 384) + tr = upd["traces"][0] + assert tr["mode"] == "points" and tr["reduction"] == "none" + (wx0, wx1), (wy0, wy1) = tr["x_range"], tr["y_range"] + # Contains the view, wider than it, still under budget. + assert wx0 <= 40.0 and wx1 >= 44.0 and wy0 <= 40.0 and wy1 >= 44.0 + assert (wx1 - wx0) > 4.0 and (wy1 - wy0) > 4.0 + assert tr["visible"] <= SCATTER_DENSITY_THRESHOLD + # visible counts the SHIPPED window exactly. + inwin = int(np.sum((x >= wx0) & (x <= wx1) & (y >= wy0) & (y <= wy1))) + assert tr["visible"] == inwin + # §16: offsets re-center on the SHIPPED window's midpoint, and the span + # cap keeps the padded window inside the client's re-encode ladder. + assert tr["x"]["offset"] == pytest.approx((wx0 + wx1) / 2.0) + assert (wx1 - wx0) <= 4.0 * DRILL_PAD_SPAN_CAP + assert (wy1 - wy0) <= 4.0 * DRILL_PAD_SPAN_CAP + + +def test_padded_windows_are_identical_across_pans() -> None: + fig, _, _ = _uniform_fig(SCATTER_DENSITY_THRESHOLD + 300_000) + upd_a, _ = fig.density_view(0, 40.0, 44.0, 40.0, 44.0, 512, 384) + upd_b, _ = fig.density_view(0, 40.7, 44.7, 39.5, 43.5, 512, 384) + tr_a, tr_b = upd_a["traces"][0], upd_b["traces"][0] + assert tr_a["mode"] == tr_b["mode"] == "points" + # A small pan inside the same aligned blocks resolves to the SAME shipped + # window — the client-side cache dedupes it, the wire carries it once. + assert tr_a["x_range"] == tr_b["x_range"] + assert tr_a["y_range"] == tr_b["y_range"] + + +def test_padding_falls_back_to_view_window_when_over_budget() -> None: + # A view centered on the extent midpoint straddles a grid line at every + # candidate level, so ANY aligned superset roughly doubles the count; with + # the view's own count near the budget, every ladder rung is over budget + # and the raw view window ships, exactly as before padding existed. + n = 1_000_000 + rng = np.random.default_rng(5) + x = rng.uniform(0.0, 100.0, n) + y = rng.uniform(0.0, 100.0, n) + fig = Figure().scatter(x, y, density=True) + # view count ~= n * (a/100)^2 targeted at ~0.95 * budget, centered at 50. + a = 100.0 * float(np.sqrt(0.95 * SCATTER_DENSITY_THRESHOLD / n)) + upd, _ = fig.density_view(0, 50.0 - a / 2, 50.0 + a / 2, 50.0 - a / 2, 50.0 + a / 2, 512, 384) + tr = upd["traces"][0] + assert tr["mode"] == "points" + assert tr["x_range"] == [50.0 - a / 2, 50.0 + a / 2] + assert tr["y_range"] == [50.0 - a / 2, 50.0 + a / 2] + assert tr["visible"] <= SCATTER_DENSITY_THRESHOLD + + +def test_nonlinear_axis_skips_padding() -> None: + # Raw-space alignment mis-sizes log windows near zero; nonlinear-axis + # traces keep today's exact-view drill (recorded contract, LOD doc T13). + n = SCATTER_DENSITY_THRESHOLD + 100_000 + rng = np.random.default_rng(9) + x = rng.uniform(1.0, 1000.0, n) + y = rng.uniform(0.0, 100.0, n) + fig = Figure().scatter(x, y, density=True) + fig.set_axis("x", type_="log") + upd, _ = fig.density_view(0, 10.0, 20.0, 40.0, 60.0, 512, 384) + tr = upd["traces"][0] + assert tr["mode"] == "points" + assert tr["x_range"] == [10.0, 20.0] + assert tr["y_range"] == [40.0, 60.0] + + +def test_padded_span_cap_guards_f32_reencode_ladder() -> None: + # A tiny dataset fits any window under the budget, so only the span cap + # stops a microscopic view from shipping the whole extent — whose midpoint + # offset would leave the client's §16 re-encode requests unable to ever + # improve precision. + n = 50_000 + rng = np.random.default_rng(13) + x = rng.uniform(0.0, 100.0, n) + y = rng.uniform(0.0, 100.0, n) + fig = Figure().scatter(x, y, density=True) + upd, _ = fig.density_view(0, 50.0, 50.001, 50.0, 50.001, 512, 384) + tr = upd["traces"][0] + assert tr["mode"] == "points" + (wx0, wx1), (wy0, wy1) = tr["x_range"], tr["y_range"] + assert wx0 <= 50.0 and wx1 >= 50.001 + assert (wx1 - wx0) <= 0.001 * DRILL_PAD_SPAN_CAP + assert (wy1 - wy0) <= 0.001 * DRILL_PAD_SPAN_CAP + + +def test_append_clears_drill_history() -> None: + n = SCATTER_DENSITY_THRESHOLD + 50_000 + rng = np.random.default_rng(17) + x = rng.uniform(0.0, 100.0, n) + y = rng.uniform(0.0, 100.0, n) + fig = Figure().scatter(x, y, density=True) + upd, _ = fig.density_view(0, 10.0, 16.0, 10.0, 16.0, 512, 384) + seq = upd["traces"][0]["drill_seq"] + assert fig.pick(0, 0, drill_seq=seq) is not None + fig.append(0, np.asarray([1.0]), np.asarray([1.0])) + # Remembered subsets were computed against the pre-append canonical state; + # a stale-seq pick must die rather than translate (§16 exact-or-nothing). + assert fig.traces[0].drill_history == {} + assert fig.pick(0, 0, drill_seq=seq) is None diff --git a/tests/test_drill_zoomin_elides_request.py b/tests/test_drill_zoomin_elides_request.py index b7f02025..302dc65e 100644 --- a/tests/test_drill_zoomin_elides_request.py +++ b/tests/test_drill_zoomin_elides_request.py @@ -15,6 +15,11 @@ - the drill stops being an exact live subset (dying, or a reply that did not claim `reduction: "none"`). +A re-armed request is still subject to the T13 identical-request suppression: +the same window at the same screen size never goes on the wire twice while +the first request is in flight — the trace keeps waiting on the original seq, +whose reply is then accepted instead of dying stale. + This drives the real client in headless Chromium with a captured comm, applies a drill for a known window, and inspects exactly which view requests reach the wire. @@ -102,9 +107,28 @@ g._drillDying = false; // 5) A subset that did not claim reduction "none" never arms the elision. + // Distinct bounds from case 4: an identical re-request would be elided by + // the T13 duplicate suppression asserted separately below. g.drill.exact = false; const nonExactRequested = request( - cx - sx * 0.02, cx + sx * 0.02, cy - sy * 0.02, cy + sy * 0.02) === 1; + cx - sx * 0.025, cx + sx * 0.025, cy - sy * 0.025, cy + sy * 0.025) === 1; + + // 6) Identical-request suppression (T13): re-requesting the SAME window + // and screen while the first request is still in flight sends nothing, + // and the trace keeps waiting on the ORIGINAL request's seq so that + // reply is accepted instead of dying stale — the fix for the + // "constantly re-requesting the same points" loop (#225 notes). + const densityMsgs = sent.filter((m) => m.type === "density_view"); + const firstReq = densityMsgs[densityMsgs.length - 1]; + const dupElided = request( + cx - sx * 0.025, cx + sx * 0.025, cy - sy * 0.025, cy + sy * 0.025) === 0; + const dupKeptPendingSeq = g._lodPendingSeq === firstReq.seq; + // The in-flight reply (old seq) is still accepted for this trace… + const acceptedOldSeq = view._densityReplyCurrent({ + seq: firstReq.seq, traces: [{ id: g.trace.id }] }); + // …but a reply for a seq nobody is waiting on stays stale (T5). + const rejectedUnknownSeq = view._densityReplyCurrent({ + seq: firstReq.seq + 1000, traces: [{ id: g.trace.id }] }); document.body.setAttribute("data-xy-elide-probe", JSON.stringify({ hasDensity: !!g, @@ -117,6 +141,10 @@ deepZoomRequested, dyingRequested, nonExactRequested, + dupElided, + dupKeptPendingSeq, + acceptedOldSeq, + rejectedUnknownSeq, })); } catch (err) { document.body.setAttribute( @@ -176,3 +204,11 @@ def test_zoom_in_within_exact_drill_sends_no_request(tmp_path: Path) -> None: assert result["deepZoomRequested"] is True assert result["dyingRequested"] is True assert result["nonExactRequested"] is True + # Identical-request suppression (T13): the same window+screen sends + # nothing while the original request is in flight; the trace keeps + # waiting on the original seq, whose reply is accepted — any other + # losing seq stays stale (T5). + assert result["dupElided"] is True + assert result["dupKeptPendingSeq"] is True + assert result["acceptedOldSeq"] is True + assert result["rejectedUnknownSeq"] is False diff --git a/tests/test_lod.py b/tests/test_lod.py index e8b641c4..297af624 100644 --- a/tests/test_lod.py +++ b/tests/test_lod.py @@ -136,7 +136,9 @@ def wrapped_exit(trace): assert len(calls["request"]) == 3 assert any(plan.exact is False for plan in calls["plan"]) assert any(plan.exact is True for plan in calls["plan"]) - assert len(calls["encode"]) >= 2 + # One geometry encode: the drill's window-centered x/y pair. Density + # replies ship no sample overlay (#225), so nothing else encodes points. + assert len(calls["encode"]) == 1 assert calls["enter"] == [drilled["traces"][0]["visible"]] assert True in calls["exit"] diff --git a/tests/test_scatter.py b/tests/test_scatter.py index a004e7b3..eab4091c 100644 --- a/tests/test_scatter.py +++ b/tests/test_scatter.py @@ -617,7 +617,11 @@ def test_continuous_drill_update_ships_u8_but_build_payload_stays_f32() -> None: assert spec["dtype"] == "u8" assert len(buffers[spec["buf"]]) == tr["visible"] # 1 byte/point sbuf = np.frombuffer(buffers[tr["size"]["buf"]], dtype=np.uint8) - vis = (x >= 0) & (x <= 5) & (y >= 0) & (y <= 5) + # The reply ships a padded aligned window around the view (LOD doc T13); + # the subset is every point of the SHIPPED window, in canonical row order. + (wx0, wx1), (wy0, wy1) = tr["x_range"], tr["y_range"] + assert wx0 <= 0.0 and wx1 >= 5.0 and wy0 <= 0.0 and wy1 >= 5.0 + vis = (x >= wx0) & (x <= wx1) & (y >= wy0) & (y <= wy1) expected = (s[vis] - s.min()) / (s.max() - s.min()) np.testing.assert_allclose(sbuf / 255.0, expected, atol=0.5 / 255.0 + 1e-6) @@ -679,14 +683,12 @@ def test_density_view_rebins(): inwin = np.sum((x >= 10) & (x < 90) & (y >= 20) & (y < 80)) assert inwin > SCATTER_DENSITY_THRESHOLD # really over budget assert grid.sum() == pytest.approx(inwin, rel=0.05) - sample = d["sample"] - assert sample["mode"] == "sampled" - assert sample["visible"] == update["traces"][0]["visible"] == inwin - assert 0 < sample["n"] <= int(DENSITY_SAMPLE_TARGET * 1.2) - xs = np.frombuffer(buffers[sample["x"]["buf"]], dtype=np.float32) - ys = np.frombuffer(buffers[sample["y"]["buf"]], dtype=np.float32) - assert len(xs) == sample["n"] - assert len(ys) == sample["n"] + # Interactive density replies ship NO point sample (#225): above the drill + # budget a fixed-size sample reads as individual data points at a zoom + # where real points are sub-pixel. The only retained sample is the + # first-payload one, drawn below the resolvable-count gate client-side. + assert "sample" not in d + assert update["traces"][0]["visible"] == inwin def test_density_view_coarsens_sparse_screen_grid(): @@ -723,12 +725,21 @@ def test_density_view_drills_to_points_when_window_fits(): assert tr["mode"] == "points" assert tr["tier"] == "direct" assert tr["reduction"] == "none" - inwin = int(np.sum((x >= 0) & (x <= 10) & (y >= 0) & (y <= 10))) - assert 0 < inwin <= SCATTER_DENSITY_THRESHOLD + view_inwin = int(np.sum((x >= 0) & (x <= 10) & (y >= 0) & (y <= 10))) + assert 0 < view_inwin <= SCATTER_DENSITY_THRESHOLD + # The shipped window is a padded ALIGNED superset of the view (LOD doc + # T13) so nearby pans/zooms render from the client's point-window cache; + # the subset is every point of that window, still within the budget. + (wx0, wx1), (wy0, wy1) = tr["x_range"], tr["y_range"] + assert wx0 <= 0.0 and wx1 >= 10.0 and wy0 <= 0.0 and wy1 >= 10.0 + vis = (x >= wx0) & (x <= wx1) & (y >= wy0) & (y <= wy1) + inwin = int(vis.sum()) + assert view_inwin <= inwin <= SCATTER_DENSITY_THRESHOLD assert tr["visible"] == inwin and tr["x"]["len"] == inwin xs = np.frombuffer(bufs[tr["x"]["buf"]], dtype=np.float32) assert len(xs) == inwin - assert tr["x"]["offset"] == pytest.approx(5.0) # §16 window-centered + # §16 window-centered offset — the SHIPPED window's midpoint. + assert tr["x"]["offset"] == pytest.approx((wx0 + wx1) / 2.0) # Continuous channels on the drill wire are u8 LUT coordinates (§29: # live-interaction readbacks resolve server-side, so quantization is # invisible; the ramp/colormap only has ~256 useful levels anyway). @@ -738,23 +749,23 @@ def test_density_view_drills_to_points_when_window_fits(): assert fig.traces[0].drill_mode is True # pick speaks drilled indices: shipped 0 -> a canonical row in the window row = fig.pick(0, 0) - assert row is not None and 0.0 <= row["x"] <= 10.0 + assert row is not None and wx0 <= row["x"] <= wx1 assert "color_value" in row # Intensity handoff (§5): per-point local log-density (u8, like every - # unit-scalar live channel) and a blend weight = visible/budget. Freshly - # drilled points enter at their cell's count-alpha and ease to native - # opacity; the surface wears the mean point color (LOD doc §2), so no - # density_colormap rides the points wire. + # unit-scalar live channel) and a blend weight keyed on the VIEW's own + # count — padding widens what ships, not what the user is looking at. + # Freshly drilled points enter at their cell's count-alpha and ease to + # native opacity; the surface wears the mean point color (LOD doc §2), so + # no density_colormap rides the points wire. assert tr["density_val"]["dtype"] == "u8" dbuf = np.frombuffer(bufs[tr["density_val"]["buf"]], dtype=np.uint8) assert len(dbuf) == inwin assert dbuf.max() == 255 # the hottest cell reaches full handoff alpha - assert tr["lod_blend"] == pytest.approx(inwin / SCATTER_DENSITY_THRESHOLD) + assert tr["lod_blend"] == pytest.approx(view_inwin / SCATTER_DENSITY_THRESHOLD) assert "density_colormap" not in tr # Channels are normalized over the *global* domain after slicing (staff # review: slice-first must not change values — colors stay view-stable), # then quantized: every byte within half a step of the exact unit value. - vis = (x >= 0) & (x <= 10) & (y >= 0) & (y <= 10) expected = (c[vis] - c.min()) / (c.max() - c.min()) np.testing.assert_allclose(cbuf / 255.0, expected, atol=0.5 / 255.0 + 1e-6) @@ -823,9 +834,11 @@ def fake_bin_2d(_x, _y, _lo_x, _hi_x, _lo_y, _hi_y, w, h): def test_drill_seq_guards_stale_picks(): - # A pick that raced a drill update must return None — never a row read - # through the wrong index space (§16: exact or nothing). - # n chosen so the full window (n visible) clears the 1.15x hysteresis exit. + # A pick against another subset version must translate through THAT + # subset when it is still remembered (LOD doc T13: the client can serve a + # view from a retired cached point window) and return None once it is not + # — never a row read through the wrong index space (§16: exact or + # nothing). n chosen so the full window clears the 1.15x hysteresis exit. n = SCATTER_DENSITY_THRESHOLD + 50_000 rng = np.random.default_rng(11) x = rng.uniform(0, 100, n) @@ -833,22 +846,48 @@ def test_drill_seq_guards_stale_picks(): fig = Figure().scatter(x, y, density=True) upd1, _ = fig.density_view(0, 0.0, 10.0, 0.0, 10.0, 512, 384) seq1 = upd1["traces"][0]["drill_seq"] + sel1 = fig.traces[0].shipped_sel.copy() assert seq1 == 1 - assert fig.pick(0, 0, drill_seq=seq1) is not None # matching subset: exact row + row1 = fig.pick(0, 0, drill_seq=seq1) # matching subset: exact row + assert row1 is not None and row1["index"] == int(sel1[0]) assert fig.pick(0, 0) is not None # legacy caller without seq still works assert fig.pick(0, 0, drill_seq=True) is None # bool must not alias seq 1 upd2, _ = fig.density_view(0, 20.0, 30.0, 20.0, 30.0, 512, 384) seq2 = upd2["traces"][0]["drill_seq"] assert seq2 == seq1 + 1 - assert fig.pick(0, 0, drill_seq=seq1) is None # stale subset: dropped + # The previous subset stays remembered (bounded history): a pick against + # it translates through THAT subset — the exact row it named, not the new + # subset's row 0 — because the client may still be drawing that window. + row_old = fig.pick(0, 0, drill_seq=seq1) + assert row_old is not None and row_old["index"] == int(sel1[0]) assert fig.pick(0, 0, drill_seq=seq2) is not None - - # Drill-out bumps the version too: a drilled-index pick arriving after the - # subset died must not be read as a *canonical* index. + # An index past the remembered subset's length is dropped, and a seq the + # history never held (bumped by exits, or simply unknown) is dropped too. + assert fig.pick(0, len(sel1), drill_seq=seq1) is None + assert fig.pick(0, 0, drill_seq=seq2 + 50) is None + + # Drill-out keeps remembered subsets alive for retired client windows, + # but the exit-bumped seq itself never shipped a subset: a pick against + # it must not be read as a *canonical* index. upd3, _ = fig.density_view(0, 0.0, 100.0, 0.0, 100.0, 512, 384) assert upd3["traces"][0]["mode"] == "density" - assert fig.pick(0, 0, drill_seq=seq2) is None + exit_seq = fig.traces[0].drill_seq + assert exit_seq == seq2 + 1 + assert fig.pick(0, 0, drill_seq=exit_seq) is None + row_after_exit = fig.pick(0, 0, drill_seq=seq2) + (wx0, wx1) = upd2["traces"][0]["x_range"] # seq2 shipped its PADDED window + assert row_after_exit is not None and wx0 <= row_after_exit["x"] <= wx1 + + # The history is bounded: churning past DRILL_HISTORY_KEEP subsets expires + # the oldest, whose picks then drop instead of translating. + from xy.config import DRILL_HISTORY_KEEP + + for i in range(DRILL_HISTORY_KEEP + 1): + lo = 5.0 * (i % 8) + upd_i, _ = fig.density_view(0, lo, lo + 6.0, lo, lo + 6.0, 512, 384) + assert upd_i["traces"][0]["mode"] == "points" + assert fig.pick(0, 0, drill_seq=seq1) is None # out-of-range trace ids are rejected, not wrapped pythonically assert fig.pick(-1, 0) is None assert fig.pick(99, 0) is None