From cf0bd89da2253728017f4a4bcf1f8f53d1b0dd62 Mon Sep 17 00:00:00 2001 From: lbwexler Date: Fri, 18 Sep 2026 15:57:19 -0400 Subject: [PATCH 1/5] Add derived fields (FieldSpec.derivedFn) for Stores and Cube Views * Field.derivedFn computes a value from the record's other values, named in the required dependsOn, via a getter on record data - never loaded, parsed or written. * On a CubeField the function also runs on every View row where the field is not aggregated: with an aggregator it derives leaves and rolls up, without one it derives each level from that row's aggregates. A Query including a derived field includes its inputs. * Stores connected to a Cube View are now always projectionOnly - the View sets the flag, and conflicting config throws. --- CHANGELOG.md | 8 ++++ data/Field.ts | 49 ++++++++++++++++++++-- data/README.md | 21 ++++++++-- data/Store.ts | 65 +++++++++++++++++++----------- data/StoreRecord.ts | 3 +- data/cube/Cube.ts | 1 + data/cube/CubeField.ts | 6 +++ data/cube/Query.ts | 20 ++++++++- data/cube/README.md | 25 ++++++++++++ data/cube/View.ts | 31 ++++++++++---- data/cube/impl/RowDataGenerator.ts | 48 ++++++++++++++++------ 11 files changed, 226 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8259aad37b..6bdd1a957c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,9 @@ Prefer the new `GridModel.theme` config (below) or the `--xh-grid-*` variables over CSS wherever they suffice. +* Stores connected to a Cube `View` are now always `projectionOnly` - the View sets the flag, and + an explicit `false` or a `processRawData` config on a connected store throws. Apps parsing View + rows into their own records must instead declare the needed fields on the Cube. * Scheduled Removals * Removed `HoistBase.withSpan()`, deprecated in v86. Use `runner().span(...)` to start a `Runner` chain instead. Note that `TraceService.withSpan()` remains available for advanced use. @@ -52,6 +55,11 @@ ### 🎁 New Features +* Added `FieldSpec.derivedFn` - a field computed from the record's other values, named in the + required `dependsOn`, and read through a getter so it is always current. On a `CubeField` the + function also runs on every View row where the field is not aggregated: with an `aggregator` it + derives each leaf and rolls up (market value), without one it derives each level from that + row's aggregates (PnL in bps). A Query including a derived field includes its inputs. * Added a `theme` config to `GridModel` and `AgGridModel`, accepting AG Grid theme param overrides (e.g. `{headerBackgroundColor: 'navy', spacing: 4}`) for grids that need to depart from the app's standard styling. Overrides are applied on top of Hoist's own theme, so grids keep their bindings diff --git a/data/Field.ts b/data/Field.ts index 8ba3a81763..18654d09b5 100644 --- a/data/Field.ts +++ b/data/Field.ts @@ -5,10 +5,10 @@ * Copyright © 2026 Extremely Heavy Industries Inc. */ -import {XH} from '@xh/hoist/core'; +import {PlainObject, XH} from '@xh/hoist/core'; import {RuleLike} from '@xh/hoist/data/validation/Types'; import {isLocalDate, LocalDate} from '@xh/hoist/utils/datetime'; -import {withDefault} from '@xh/hoist/utils/js'; +import {throwIf, withDefault} from '@xh/hoist/utils/js'; import {Rule} from './validation/Rule'; import equal from 'fast-deep-equal'; import {isDate, isString, toNumber, isFinite, startCase, isFunction, castArray} from 'lodash'; @@ -61,8 +61,36 @@ export interface FieldSpec { * building secured internal apps with large datasets and tight performance tolerances. */ enableXssProtection?: boolean; + + /** + * Function computing this field's value from the record's other values, making it a *derived* + * field. Values are read through a getter on record `data` - never loaded, parsed, or written, + * and always current with their inputs: + * + * ```ts + * {name: 'marketValue', dependsOn: ['quantity', 'price'], derivedFn: d => d.quantity * d.price} + * ``` + * + * Requires `dependsOn`. Derived fields are read-only - {@link Store.modifyRecords} ignores + * writes to them. A Store that is a `projectionOnly` view of another's data (e.g. one + * connected to a Cube View) adopts derived values from its provider rather than computing them. + * + * On a {@link CubeField}, the function also runs on every View row where the field is not + * aggregated - so a field with an `aggregator` derives at the leaves and rolls up (market + * value), while one without derives at every level from that row's aggregates (PnL in bps). + */ + derivedFn?: DerivedFn; + + /** + * Names of the fields a `derivedFn` reads - required with `derivedFn`, and may be empty. A + * Cube Query including a derived field includes these as well. + */ + dependsOn?: string[]; } +/** Function computing a derived field's value from the other values on a record or View row. */ +export type DerivedFn = (data: PlainObject) => any; + /** * Metadata for an individual data field within a {@link StoreRecord}. * @@ -73,6 +101,11 @@ export class Field { return true; } + /** True if this field's value is computed from other values - see {@link FieldSpec.derivedFn}. */ + get isDerived(): boolean { + return !!this.derivedFn; + } + readonly name: string; readonly type: FieldType; readonly displayName: string; @@ -81,6 +114,8 @@ export class Field { readonly isDimension: boolean; readonly rules: Rule[]; readonly enableXssProtection: boolean; + readonly derivedFn: DerivedFn; + readonly dependsOn: string[]; constructor({ name, @@ -90,7 +125,9 @@ export class Field { defaultValue = null, isDimension = false, rules = [], - enableXssProtection = XH.appSpec.enableXssProtection + enableXssProtection = XH.appSpec.enableXssProtection, + derivedFn = null, + dependsOn = null }: FieldSpec) { this.name = name; this.type = type; @@ -100,6 +137,12 @@ export class Field { this.isDimension = isDimension; this.rules = this.processRuleSpecs(rules); this.enableXssProtection = enableXssProtection; + this.derivedFn = derivedFn; + this.dependsOn = dependsOn; + throwIf( + derivedFn && !dependsOn, + `Field '${name}' declares a 'derivedFn' but no 'dependsOn' - name the fields it reads, or pass [].` + ); } parseVal(val: any): any { diff --git a/data/README.md b/data/README.md index 38f6daf5f3..aa923f48ca 100644 --- a/data/README.md +++ b/data/README.md @@ -432,6 +432,21 @@ apps with large datasets. Set `enableXssProtection` per field, or app-wide via | `'tags'` | String array | Splits comma-separated | | `'pwd'` | Password | Marks as sensitive | +### Derived Fields + +A field with a `derivedFn` computes its value from the record's other values, named in the +required `dependsOn`. Values are read through a getter on record `data` - never loaded, parsed or +written - so they are always current with their inputs, and sort, filter and export like any other +field. Writes to one via `modifyRecords()` are ignored. + +```typescript +{name: 'marketValue', dependsOn: ['quantity', 'price'], derivedFn: d => d.quantity * d.price} +``` + +A `projectionOnly` store adopts derived values from its provider rather than computing them. See +the [Cube README](cube/README.md#derived-fields) for derived `CubeField`s, which also run on +aggregated View rows. + ## Filter System **Files**: `filter/Filter.ts`, `filter/FieldFilter.ts`, `filter/CompoundFilter.ts`, `filter/FunctionFilter.ts` @@ -960,9 +975,9 @@ parses and owns. Store then uses each incoming raw object *as* its record's `dat This collapses the usual two objects per row to one, and skips the per-row parse on every load and update. -Use this config for stores connected to a Cube `View`, or fed by an endpoint that returns data in -its final client-side form. A View logs a warning when a connected store leaves the config unset. -Set it explicitly to `false` to opt out and silence that warning. +Use this config for stores fed by an endpoint that returns data in its final client-side form. +Stores connected to a Cube `View` are always projections - the View sets the flag, and an explicit +`false` throws. ```typescript const store = new Store({ diff --git a/data/Store.ts b/data/Store.ts index 77e1e2868f..b476ac6351 100644 --- a/data/Store.ts +++ b/data/Store.ts @@ -196,10 +196,8 @@ export interface StoreConfig { /** * True to mark this store as a read-only projection of data owned and parsed elsewhere. - * Recommended for stores connected to a Cube {@link View} for improved performance, when no - * additional record parsing or local data modification is required. Default null - a View - * logs a warning when its connected stores leave this unset. Set explicitly to `false` to - * opt out and silence the warning. + * Default null. Stores connected to a Cube {@link View} are always projections - the View + * sets this flag, and an explicit `false` throws. * * Each incoming raw object is used *as* its record's `data`, by reference, skipping the * per-record parse and copy on every load and update. Raw data must already match what the @@ -372,7 +370,7 @@ export class Store idEncodesTreePath: boolean; freezeData: boolean; retainRaw: boolean; - readonly projectionOnly: boolean; + projectionOnly: boolean; validationIsComplex: boolean; @observable.ref @@ -411,8 +409,9 @@ export class Store @observable.ref _filtered: RecordSet; - private _dataTemplate: PlainObject = null; private _dataDefaults: PlainObject = null; + private _dataTemplate: PlainObject = null; + private _denseDataProto: PlainObject = null; private _denseRecordThreshold: number; private _digestSpec: StoreRecordDigestSpec; private _digestFn: (raw: PlainObject) => StoreRecordDigest; @@ -485,6 +484,7 @@ export class Store this._fieldMap = this.createFieldMap(); this._dataDefaults = this.createDataDefaults(); this._dataTemplate = {...this._dataDefaults}; // Clone for fast-props mode. + this._denseDataProto = this.createDenseDataProto(); this._denseRecordThreshold = this.experimental.denseRecordThreshold ?? DENSE_RECORD_THRESHOLD; if (data) this.loadData(data); @@ -1525,7 +1525,7 @@ export class Store rescuable = !!cached; for (const name in data) { const field = _fieldMap.get(name); - if (field) { + if (field && !field.isDerived) { const val = field.parseVal(data[name]); if (val !== field.defaultValue) { if (rescuable) { @@ -1547,12 +1547,12 @@ export class Store private parseUpdate(data: PlainObject, update: PlainObject): PlainObject { // Merge updated values over current ones, then rebuild exactly as parseOrRescue() would. const {_recordBuildData} = this, - {names, vals} = _recordBuildData, - hasOwn = Object.prototype.hasOwnProperty; + {names, vals} = _recordBuildData; let n = 0; this.fields.forEach(field => { + if (field.isDerived) return; const {name} = field, - val = hasOwn.call(update, name) ? field.parseVal(update[name]) : data[name]; + val = Object.hasOwn(update, name) ? field.parseVal(update[name]) : data[name]; if (val !== field.defaultValue) { names[n] = name; vals[n] = val; @@ -1575,18 +1575,23 @@ export class Store * - At or above it, a clone of the shared template carrying every Field. Wide objects built * by per-property adds are demoted to V8's dictionary mode - cloning sidesteps the adds * (overwriting an existing property is not an add), so all dense records share the - * template's one fixed shape. + * template's one fixed shape. With derived fields, the clone also takes `_dataDefaults` + * as its prototype to reach their getters - a slower clone, so only paid when needed. * * The representation is decided per record, from parsed content alone - records with equal * field values always take equal shapes, which the deep-equal comparisons in modifyRecords() * require. */ private buildData(): PlainObject { - const {names, vals, n} = this._recordBuildData, + // Literal `__proto__` key sets the prototype at creation (ES Annex B). + const {_denseRecordThreshold, _denseDataProto, _dataTemplate, _dataDefaults} = this, + {names, vals, n} = this._recordBuildData, ret = - n >= this._denseRecordThreshold - ? {...this._dataTemplate} - : Object.create(this._dataDefaults); + n >= _denseRecordThreshold + ? _denseDataProto + ? {__proto__: _denseDataProto, ..._dataTemplate} + : {..._dataTemplate} + : Object.create(_dataDefaults); for (let i = 0; i < n; i++) { ret[names[i]] = vals[i]; } @@ -1600,16 +1605,30 @@ export class Store ); } - /** - * Shared template for record `data` objects - an own property for every Field, holding its - * defaultValue. `parseOrRescue()` clones it per record, so all records in a Store share one - * identical, fixed shape. That keeps them in V8's compact fast-properties mode: objects built - * instead by per-field property adds are demoted to a per-object hashtable ("dictionary mode") - * past ~20 adds, costing several times more memory per record. - */ + /** Prototype for all record `data` - a defaultValue per stored Field, plus derived getters. */ private createDataDefaults() { const ret = {}; - this.fields.forEach(({name, defaultValue}) => (ret[name] = defaultValue)); + this.fields.forEach(({name, defaultValue, isDerived}) => { + if (!isDerived) ret[name] = defaultValue; + }); + return ret; + } + + // Install derived getters on `_dataDefaults`, returning it as the dense-record prototype - + // null without derived fields, leaving dense records prototype-free. + private createDenseDataProto(): PlainObject { + const derived = this.fields.filter(it => it.isDerived); + if (isEmpty(derived)) return null; + const ret = this._dataDefaults; + derived.forEach(({name, derivedFn}) => { + Object.defineProperty(ret, name, { + get(this: PlainObject) { + return derivedFn(this); + }, + enumerable: true, + configurable: true + }); + }); return ret; } diff --git a/data/StoreRecord.ts b/data/StoreRecord.ts index 9c6f097f58..d7ef6bf972 100644 --- a/data/StoreRecord.ts +++ b/data/StoreRecord.ts @@ -248,7 +248,8 @@ export class StoreRecord { const {data, committedData} = this, ret: PlainObject = {}; - this.fields.forEach(({name}) => { + this.fields.forEach(({name, isDerived}) => { + if (isDerived) return; const val = data[name]; if (!equal(val, committedData[name])) ret[name] = val; }); diff --git a/data/cube/Cube.ts b/data/cube/Cube.ts index 99e66d6c9a..669c422941 100755 --- a/data/cube/Cube.ts +++ b/data/cube/Cube.ts @@ -282,6 +282,7 @@ export class Cube extends HoistBase { * @param query - query to be used to construct this view. * @param stores - Stores to be automatically loaded/reloaded with View results. * @param connect - true to update View automatically when data in the underlying Cube changes. + * @param xhName - see {@link HoistBase.xhName}. */ createView({ query, diff --git a/data/cube/CubeField.ts b/data/cube/CubeField.ts index 0fd5fa177c..416eb4656c 100755 --- a/data/cube/CubeField.ts +++ b/data/cube/CubeField.ts @@ -23,6 +23,7 @@ import { SumStrictAggregator, UniqueAggregator } from '@xh/hoist/data'; +import {throwIf} from '@xh/hoist/utils/js'; import {isString} from 'lodash'; export interface CubeFieldSpec extends FieldSpec { @@ -128,6 +129,11 @@ export class CubeField extends Field { // Dimension specific this.isLeafDimension = isLeafDimension; this.parentDimension = parentDimension; + + throwIf( + this.isDerived && this.isDimension, + `CubeField '${this.name}' may not be both derived and a dimension.` + ); } //------------------------ diff --git a/data/cube/Query.ts b/data/cube/Query.ts index cd46cbcb0c..ce61d17efb 100755 --- a/data/cube/Query.ts +++ b/data/cube/Query.ts @@ -173,7 +173,7 @@ export class Query { this.dimensions = this.parseDimensions(dimensions); // Ensure canonical field order so equivalent queries compare equal this.fields = sortBy( - uniq([...this.parseFields(fields), ...(this.dimensions ?? [])]), + uniq(this.withDependencies([...this.parseFields(fields), ...(this.dimensions ?? [])])), 'name' ); this.includeRoot = includeRoot; @@ -252,6 +252,24 @@ export class Query { return fields.filter(f => names.includes(f.name)); } + private withDependencies(fields: CubeField[]): CubeField[] { + const ret = [...fields], + names = new Set(fields.map(it => it.name)); + for (let i = 0; i < ret.length; i++) { + ret[i].dependsOn?.forEach(name => { + if (names.has(name)) return; + const field = find(this.cube.fields, {name}); + throwIf( + !field, + `Field '${ret[i].name}' depends on '${name}', which is not a Field on this Cube.` + ); + names.add(name); + ret.push(field); + }); + } + return ret; + } + private parseDimensions(raw: CubeField[] | string[]): CubeField[] { if (!raw) return null; if (raw[0] instanceof CubeField) return raw.slice() as CubeField[]; // force clone, we retain. diff --git a/data/cube/README.md b/data/cube/README.md index f07f79161a..1b12ec77f6 100644 --- a/data/cube/README.md +++ b/data/cube/README.md @@ -165,6 +165,31 @@ Rules to observe: that depend on values beyond their own children (e.g. percent-of-total) must return false for the same reason, and doing so also gives them access to `AggregationContext.filteredRecords`. +## Derived Fields + +A field with a `derivedFn` computes its value from the row's other values wherever the field is +not aggregated. `dependsOn` names the fields it reads and is required - a Query including a derived +field includes its inputs as well. See `FieldSpec.derivedFn` in `data/README.md` for the Store-level +form, which the Cube's own store uses to derive every leaf. + +```typescript +// Derived at each leaf, then summed - a product belongs at the leaf. +{name: 'notional', aggregator: 'SUM', dependsOn: ['qty', 'price'], derivedFn: d => d.qty * d.price}, + +// Derived at every level from that row's sums - a ratio belongs at the level. +{name: 'pnlBps', dependsOn: ['pnl', 'notional'], derivedFn: d => (d.pnl / d.notional) * 10000}, + +// Derived at leaves only - parents publish null. +{name: 'side', aggregator: 'NULL', dependsOn: ['qty'], derivedFn: d => (d.qty > 0 ? 'Buy' : 'Sell')} +``` + +A derived field with an aggregator is an ordinary measure whose leaf values are computed rather +than loaded. One without an aggregator is read through a getter on each parent row, so it is +always current with that row's aggregates. Derived fields may read other derived fields. + +Stores connected to a View adopt these values from the rows they receive and compute nothing +themselves - connected stores are always `projectionOnly`. + ## Querying with Views Views are the primary interface for consuming Cube data. Create them via `Cube.createView()` diff --git a/data/cube/View.ts b/data/cube/View.ts index de02580c79..3ef58764a3 100755 --- a/data/cube/View.ts +++ b/data/cube/View.ts @@ -51,9 +51,8 @@ export interface ViewConfig { * Store(s) to be automatically (re)loaded with data from this view. * Optional - read {@link View.result} directly to use without a Store. * - * Connected stores should generally set {@link StoreConfig.projectionOnly} - view rows are - * already parsed and owned by this View, so adopting them directly improves performance - * when no additional record parsing or local data modification is required. + * Connected stores are read-only projections of this View's rows - the View sets + * {@link StoreConfig.projectionOnly} on them, and conflicting config throws. */ stores?: Store[] | Store; @@ -160,6 +159,8 @@ export class View // that are not themselves an applied dimension there - and useful subsets of same. Indexed by // row depth, with entry 0 (no dimensions applied) holding the superset for the whole query. _aggFieldsByDepth: CubeField[][] = null; + // Derived fields without an aggregator - computed by getter on every row from its aggregates. + _levelDerivedFields: CubeField[] = null; _aggFieldNamesByDepth: Set[] = null; _canAggregateFnFieldsByDepth: CubeField[][] = null; _complexAggFieldsByDepth: CubeField[][] = null; @@ -361,6 +362,7 @@ export class View private buildIndices() { this._fieldsByName = new Map(this.fields.map(it => [it.name, it])); + this._levelDerivedFields = this.fields.filter(it => it.isDerived && !it.aggregator); // Aggregation eligibility is a function of level alone - dimensions apply in order, and // bucket rows share the level of the aggregate row above them. Note depth 0 has no applied @@ -422,6 +424,19 @@ export class View updatedRowDatas.forEach(rowData => this.assignDigest(rowData)); + // Level-derived values move with their inputs, with no leaf-level diff to report them. + if (changedFields.size) { + for (let added = true; added;) { + added = false; + this._levelDerivedFields.forEach(({name, dependsOn}) => { + if (!changedFields.has(name) && dependsOn.some(it => changedFields.has(it))) { + changedFields.add(name); + added = true; + } + }); + } + } + this.createAggregationContext(); stores.forEach(store => { @@ -717,11 +732,11 @@ export class View '`Store.idEncodesTreePath` cannot be configured on a Store connected to a Cube View - view row ids do not encode a fixed tree position. Leave unset.' ); - if (ret.some(s => s.projectionOnly == null && !s.processRawData)) { - this.logWarn( - 'Connected store(s) do not set `projectionOnly` - recommended for improved performance when no additional record parsing or local data modification is required. Set explicitly to false to opt out and silence this warning.' - ); - } + throwIf( + ret.some(s => s.projectionOnly === false || s.processRawData), + 'A Store connected to a Cube View is a read-only projection of its rows - remove conflicting `projectionOnly: false` or `processRawData` config.' + ); + ret.forEach(s => (s.projectionOnly = true)); return ret; } diff --git a/data/cube/impl/RowDataGenerator.ts b/data/cube/impl/RowDataGenerator.ts index 64bfd08454..8b09acb0d0 100644 --- a/data/cube/impl/RowDataGenerator.ts +++ b/data/cube/impl/RowDataGenerator.ts @@ -6,7 +6,7 @@ */ import {PlainObject} from '@xh/hoist/core'; -import {isEqual} from 'lodash'; +import {isEmpty, isEqual} from 'lodash'; import type {View} from '../View'; import {ViewRowData} from '../ViewRowData'; @@ -20,8 +20,9 @@ import {ViewRowData} from '../ViewRowData'; * Row shapes are fixed per query, keeping them in V8's compact fast-properties mode rather than * "dictionary mode": * - Aggregate and bucket row data is cloned from a shared template carrying a slot for every - * ViewRowData property and query field. Rows are only ever written via overwrites of these - * slots - never property adds. + * ViewRowData property and aggregated query field. Rows are only ever written via overwrites + * of these slots - never property adds. Derived fields with no aggregator hold no slot - when + * present, the clone takes a shared prototype whose getters compute them from the row. * - Exposed-leaf row data holds no per-leaf copy of field values - queried fields are read * through prototype getters over an own `_src` reference to the leaf's cube record data. One * generated class per query keeps all leaf datas on a single shape with monomorphic, @@ -33,8 +34,9 @@ export class RowDataGenerator { private view: View; private fieldNames: string[]; private exposesLeaves: boolean; - private parentDataTemplate: ViewRowData = null; - private leafDataClass: LeafDataClass = null; + private parentTemplate: ViewRowData = null; + private parentProto: PlainObject = null; + private leafClass: LeafDataClass = null; constructor(view: View) { this.view = view; @@ -43,12 +45,15 @@ export class RowDataGenerator { /** Create a new aggregate or bucket row data object as a clone of the shared template. */ newParentRowData(id: string): ViewRowData { - return {...this.parentDataTemplate, id}; + const {parentTemplate, parentProto} = this; + return parentProto + ? {__proto__: parentProto, ...parentTemplate, id} + : {...parentTemplate, id}; } /** Create an exposed-leaf data object - fields read via prototype getters over `src`. */ newLeafRowData(id: string, src: PlainObject): ViewRowData { - return new this.leafDataClass(id, src); + return new this.leafClass(id, src); } //------------------ @@ -67,11 +72,12 @@ export class RowDataGenerator { private init() { this.fieldNames = this.view.fieldNames; this.exposesLeaves = this.view.exposesLeaves; - this.parentDataTemplate = this.buildParentDataTemplate(); - this.leafDataClass = this.buildLeafDataClass(); + this.parentTemplate = this.buildParentTemplate(); + this.parentProto = this.buildParentProto(); + this.leafClass = this.buildLeafClass(); } - private buildParentDataTemplate(): ViewRowData { + private buildParentTemplate(): ViewRowData { const rowData: PlainObject = { id: null, cubeRowType: null, @@ -83,13 +89,31 @@ export class RowDataGenerator { cubeRowDigest: null, _cubeLeafChildren: null }; - this.view.fields.forEach(({name}) => (rowData[name] = null)); + this.view.fields.forEach(({name, isDerived, aggregator}) => { + if (!isDerived || aggregator) rowData[name] = null; + }); // Convert into V8 fast-properties mode that we'll need to mint additional fast objects return {...rowData} as ViewRowData; } - private buildLeafDataClass(): LeafDataClass { + // Getters for derived fields with no aggregato + private buildParentProto(): PlainObject { + const derived = this.view.fields.filter(it => it.isDerived && !it.aggregator); + if (isEmpty(derived)) return null; + const ret = {}; + derived.forEach(({name, derivedFn}) => { + Object.defineProperty(ret, name, { + get(this: ViewRowData) { + return derivedFn(this); + }, + enumerable: true + }); + }); + return ret; + } + + private buildLeafClass(): LeafDataClass { if (!this.exposesLeaves) return null; class LeafRowData extends BaseLeafRowData {} From 3c6cc87ceb608b3d0cf8ab3332feef79e214273a Mon Sep 17 00:00:00 2001 From: lbwexler Date: Fri, 18 Sep 2026 23:39:32 -0400 Subject: [PATCH 2/5] Build simple-proto and dense-template record data paths independently - `Store` and `RowDataGenerator` each hold one `{data, proto}` template, built in a single pass over fields with a per-field `addDerivedGetter`, replacing the shared defaults object that doubled as the dense-record prototype - Comment the fixpoint loop over level-derived fields in `View.applyDataUpdate` - Changelog wording --- CHANGELOG.md | 4 +- data/Store.ts | 104 +++++++++++++++++------------ data/cube/View.ts | 1 + data/cube/impl/RowDataGenerator.ts | 74 ++++++++++---------- 4 files changed, 101 insertions(+), 82 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cf4db31e7..31c716b1c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -75,8 +75,8 @@ * Added `FieldSpec.derivedFn` - a field computed from the record's other values, named in the required `dependsOn`, and read through a getter so it is always current. On a `CubeField` the function also runs on every View row where the field is not aggregated: with an `aggregator` it - derives each leaf and rolls up (market value), without one it derives each level from that - row's aggregates (PnL in bps). A Query including a derived field includes its inputs. + derives each leaf and rolls up (e.g. market value), without one it derives each level from that + row's aggregates (e.g. PnL in bps). A Query including a derived field includes its inputs. * Added a `theme` config to `GridModel` and `AgGridModel`, accepting AG Grid theme param overrides (e.g. `{headerBackgroundColor: 'navy', spacing: 4}`) for grids that need to depart from the app's standard styling. Overrides are applied on top of Hoist's own theme, so grids keep their bindings diff --git a/data/Store.ts b/data/Store.ts index 8df31c2485..1611d85f07 100644 --- a/data/Store.ts +++ b/data/Store.ts @@ -400,10 +400,9 @@ export class Store @observableRef private accessor _current: RecordSet; @observableRef accessor _filtered: RecordSet; - private _dataDefaults: PlainObject = null; - private _dataTemplate: PlainObject = null; - private _denseDataProto: PlainObject = null; - private _denseRecordThreshold: number; + private _simpleProto: PlainObject = null; + private _denseTemplate: {data: PlainObject; proto: PlainObject} = null; + private _denseThreshold: number; private _digestSpec: StoreRecordDigestSpec; private _digestFn: (raw: PlainObject) => StoreRecordDigest; @@ -472,11 +471,9 @@ export class Store this.validator = new StoreValidator({store: this}); this._fieldMap = this.createFieldMap(); - this._dataDefaults = this.createDataDefaults(); - this._dataTemplate = {...this._dataDefaults}; // Clone for fast-props mode. - this._denseDataProto = this.createDenseDataProto(); - this._denseRecordThreshold = - this.experimental.denseRecordThreshold ?? DENSE_RECORD_THRESHOLD; + this._simpleProto = this.createSimpleProto(); + this._denseTemplate = this.createDenseTemplate(); + this._denseThreshold = this.experimental.denseRecordThreshold ?? DENSE_RECORD_THRESHOLD; if (data) this.loadData(data); instanceManager.registerStore(this); @@ -1560,28 +1557,31 @@ export class Store * choosing its representation by their count: * * - Below `denseRecordThreshold`, a sparse object - own properties for the buffered values - * only, defaults reached through the shared `_dataDefaults` prototype. Costs nothing for - * unpopulated fields, and stays safely inside V8's fast-properties mode at these counts. - * - At or above it, a clone of the shared template carrying every Field. Wide objects built - * by per-property adds are demoted to V8's dictionary mode - cloning sidesteps the adds - * (overwriting an existing property is not an add), so all dense records share the - * template's one fixed shape. With derived fields, the clone also takes `_dataDefaults` - * as its prototype to reach their getters - a slower clone, so only paid when needed. + * only, defaults and derived getters reached through the shared `_simpleProto`. Costs + * nothing for unpopulated fields, and stays safely inside V8's fast-properties mode at + * these counts. + * - At or above it, a clone of the shared `_denseTemplate`, carrying every stored Field. + * Wide objects built by per-property adds are demoted to V8's dictionary mode - cloning + * sidesteps the adds (overwriting an existing property is not an add), so all dense + * records share the template's one fixed shape. With derived fields, the clone also takes + * the template's prototype to reach their getters - a slower clone, so only paid when + * needed. * * The representation is decided per record, from parsed content alone - records with equal * field values always take equal shapes, which the deep-equal comparisons in modifyRecords() * require. */ private buildData(): PlainObject { - // Literal `__proto__` key sets the prototype at creation (ES Annex B). - const {_denseRecordThreshold, _denseDataProto, _dataTemplate, _dataDefaults} = this, - {names, vals, n} = this._recordBuildData, - ret = - n >= _denseRecordThreshold - ? _denseDataProto - ? {__proto__: _denseDataProto, ..._dataTemplate} - : {..._dataTemplate} - : Object.create(_dataDefaults); + const {names, vals, n} = this._recordBuildData; + let ret: PlainObject; + if (n < this._denseThreshold) { + ret = Object.create(this._simpleProto); + } else { + // Literal `__proto__` key sets the prototype at creation (ES Annex B). + const {data, proto} = this._denseTemplate; + ret = proto ? {__proto__: proto, ...data} : {...data}; + } + for (let i = 0; i < n; i++) { ret[names[i]] = vals[i]; } @@ -1595,31 +1595,47 @@ export class Store ); } - /** Prototype for all record `data` - a defaultValue per stored Field, plus derived getters. */ - private createDataDefaults() { + /** Prototype for simple record `data` - a defaultValue per stored Field, plus derived getters. */ + private createSimpleProto(): PlainObject { const ret = {}; - this.fields.forEach(({name, defaultValue, isDerived}) => { - if (!isDerived) ret[name] = defaultValue; + this.fields.forEach(field => { + if (field.isDerived) { + this.addDerivedGetter(field, ret); + } else { + ret[field.name] = field.defaultValue; + } }); return ret; } - // Install derived getters on `_dataDefaults`, returning it as the dense-record prototype - - // null without derived fields, leaving dense records prototype-free. - private createDenseDataProto(): PlainObject { - const derived = this.fields.filter(it => it.isDerived); - if (isEmpty(derived)) return null; - const ret = this._dataDefaults; - derived.forEach(({name, derivedFn}) => { - Object.defineProperty(ret, name, { - get(this: PlainObject) { - return derivedFn(this); - }, - enumerable: true, - configurable: true - }); + /** + * Template for dense record `data` - an own slot per stored Field, spread-cloned per record - + * plus the prototype those clones take to reach derived getters, null without derived fields. + */ + private createDenseTemplate(): {data: PlainObject; proto: PlainObject} { + const data = {}, + proto = {}; + this.fields.forEach(field => { + if (field.isDerived) { + this.addDerivedGetter(field, proto); + } else { + data[field.name] = field.defaultValue; + } + }); + + return { + data: {...data}, // Clone for fast-props mode. + proto: isEmpty(proto) ? null : proto + }; + } + + private addDerivedGetter({name, derivedFn}: Field, target: PlainObject) { + Object.defineProperty(target, name, { + get(this: PlainObject) { + return derivedFn(this); + }, + enumerable: true }); - return ret; } private createFieldMap() { diff --git a/data/cube/View.ts b/data/cube/View.ts index 6c6a578c93..1d7b45db75 100755 --- a/data/cube/View.ts +++ b/data/cube/View.ts @@ -419,6 +419,7 @@ export class View updatedRowDatas.forEach(rowData => this.assignDigest(rowData)); // Level-derived values move with their inputs, with no leaf-level diff to report them. + // Repeat until a pass adds nothing - derived fields may depend on other derived fields. if (changedFields.size) { for (let added = true; added;) { added = false; diff --git a/data/cube/impl/RowDataGenerator.ts b/data/cube/impl/RowDataGenerator.ts index 8b09acb0d0..8c7e8f43e9 100644 --- a/data/cube/impl/RowDataGenerator.ts +++ b/data/cube/impl/RowDataGenerator.ts @@ -7,6 +7,7 @@ import {PlainObject} from '@xh/hoist/core'; import {isEmpty, isEqual} from 'lodash'; +import type {CubeField} from '../CubeField'; import type {View} from '../View'; import {ViewRowData} from '../ViewRowData'; @@ -34,8 +35,7 @@ export class RowDataGenerator { private view: View; private fieldNames: string[]; private exposesLeaves: boolean; - private parentTemplate: ViewRowData = null; - private parentProto: PlainObject = null; + private parentTemplate: {data: ViewRowData; proto: PlainObject} = null; private leafClass: LeafDataClass = null; constructor(view: View) { @@ -45,10 +45,8 @@ export class RowDataGenerator { /** Create a new aggregate or bucket row data object as a clone of the shared template. */ newParentRowData(id: string): ViewRowData { - const {parentTemplate, parentProto} = this; - return parentProto - ? {__proto__: parentProto, ...parentTemplate, id} - : {...parentTemplate, id}; + const {data, proto} = this.parentTemplate; + return proto ? {__proto__: proto, ...data, id} : {...data, id}; } /** Create an exposed-leaf data object - fields read via prototype getters over `src`. */ @@ -73,44 +71,48 @@ export class RowDataGenerator { this.fieldNames = this.view.fieldNames; this.exposesLeaves = this.view.exposesLeaves; this.parentTemplate = this.buildParentTemplate(); - this.parentProto = this.buildParentProto(); this.leafClass = this.buildLeafClass(); } - private buildParentTemplate(): ViewRowData { - const rowData: PlainObject = { - id: null, - cubeRowType: null, - cubeLabel: null, - cubeDimension: null, - cubeBuckets: null, - children: null, - isCubeLeaf: false, - cubeRowDigest: null, - _cubeLeafChildren: null - }; - this.view.fields.forEach(({name, isDerived, aggregator}) => { - if (!isDerived || aggregator) rowData[name] = null; + /** + * Template for aggregate and bucket row data - a slot per ViewRowData property and aggregated + * query field, spread-cloned per row - plus the prototype those clones take to reach the + * getters of derived fields with no aggregator, null without any. + */ + private buildParentTemplate(): {data: ViewRowData; proto: PlainObject} { + const proto = {}, + data: PlainObject = { + id: null, + cubeRowType: null, + cubeLabel: null, + cubeDimension: null, + cubeBuckets: null, + children: null, + isCubeLeaf: false, + cubeRowDigest: null, + _cubeLeafChildren: null + }; + this.view.fields.forEach(field => { + if (field.isDerived && !field.aggregator) { + this.addDerivedGetter(field, proto); + } else { + data[field.name] = null; + } }); - // Convert into V8 fast-properties mode that we'll need to mint additional fast objects - return {...rowData} as ViewRowData; + return { + data: {...data} as ViewRowData, // Clone for fast-props mode. + proto: isEmpty(proto) ? null : proto + }; } - // Getters for derived fields with no aggregato - private buildParentProto(): PlainObject { - const derived = this.view.fields.filter(it => it.isDerived && !it.aggregator); - if (isEmpty(derived)) return null; - const ret = {}; - derived.forEach(({name, derivedFn}) => { - Object.defineProperty(ret, name, { - get(this: ViewRowData) { - return derivedFn(this); - }, - enumerable: true - }); + private addDerivedGetter({name, derivedFn}: CubeField, target: PlainObject) { + Object.defineProperty(target, name, { + get(this: ViewRowData) { + return derivedFn(this); + }, + enumerable: true }); - return ret; } private buildLeafClass(): LeafDataClass { From be72335393b0a7286bb19b0bd0720a8314a3db72 Mon Sep 17 00:00:00 2001 From: lbwexler Date: Sat, 19 Sep 2026 00:00:02 -0400 Subject: [PATCH 3/5] Validate `dependsOn` at Store construction; inline parent-row getter setup - `Store.parseFields` throws on a `dependsOn` naming an unknown field, covering plain Stores and Cubes alike; `Query.withDependencies` now skips quietly - Inline the single-use derived-getter helper in `RowDataGenerator.buildParentTemplate` - Comment the level-derived `changedFields` pass in `View.applyDataUpdate` --- data/Store.ts | 22 ++++++++++++++-------- data/cube/Query.ts | 5 +---- data/cube/View.ts | 11 ++++++----- data/cube/impl/RowDataGenerator.ts | 23 +++++++++-------------- 4 files changed, 30 insertions(+), 31 deletions(-) diff --git a/data/Store.ts b/data/Store.ts index 1611d85f07..1f3bba491b 100644 --- a/data/Store.ts +++ b/data/Store.ts @@ -1346,6 +1346,12 @@ export class Store prototype of each record's data object rather than setting a value on it.` ); throwIf(uniqBy(ret, 'name').length !== ret.length, 'Field names must be unique.'); + + const names = new Set(ret.map(it => it.name)); + ret.forEach(({name, dependsOn}) => { + const missing = dependsOn?.find(it => !names.has(it)); + throwIf(missing, `Field '${name}' depends on '${missing}', which is not a Field.`); + }); return ret; } @@ -1537,13 +1543,14 @@ export class Store {names, vals} = _recordBuildData; let n = 0; this.fields.forEach(field => { - if (field.isDerived) return; - const {name} = field, - val = Object.hasOwn(update, name) ? field.parseVal(update[name]) : data[name]; - if (val !== field.defaultValue) { - names[n] = name; - vals[n] = val; - n++; + if (!field.isDerived) { + const {name} = field, + val = Object.hasOwn(update, name) ? field.parseVal(update[name]) : data[name]; + if (val !== field.defaultValue) { + names[n] = name; + vals[n] = val; + n++; + } } }); _recordBuildData.n = n; @@ -1577,7 +1584,6 @@ export class Store if (n < this._denseThreshold) { ret = Object.create(this._simpleProto); } else { - // Literal `__proto__` key sets the prototype at creation (ES Annex B). const {data, proto} = this._denseTemplate; ret = proto ? {__proto__: proto, ...data} : {...data}; } diff --git a/data/cube/Query.ts b/data/cube/Query.ts index ce61d17efb..29c9c2ce3a 100755 --- a/data/cube/Query.ts +++ b/data/cube/Query.ts @@ -259,10 +259,7 @@ export class Query { ret[i].dependsOn?.forEach(name => { if (names.has(name)) return; const field = find(this.cube.fields, {name}); - throwIf( - !field, - `Field '${ret[i].name}' depends on '${name}', which is not a Field on this Cube.` - ); + if (!field) return; names.add(name); ret.push(field); }); diff --git a/data/cube/View.ts b/data/cube/View.ts index 1d7b45db75..f6cdb86730 100755 --- a/data/cube/View.ts +++ b/data/cube/View.ts @@ -355,16 +355,17 @@ export class View } private buildIndices() { - this._fieldsByName = new Map(this.fields.map(it => [it.name, it])); - this._levelDerivedFields = this.fields.filter(it => it.isDerived && !it.aggregator); + const {fields, query} = this; + this._fieldsByName = new Map(fields.map(it => [it.name, it])); + this._levelDerivedFields = fields.filter(it => it.isDerived && !it.aggregator); // Aggregation eligibility is a function of level alone - dimensions apply in order, and // bucket rows share the level of the aggregate row above them. Note depth 0 has no applied // dimensions, and so holds the unfiltered superset of each list. Queries need not specify // dimensions at all (e.g. a leaves-only or root-total-only query) - Query.dimensions is // null in that case, leaving only the depth-0 entry below. - const dimensions = this.query.dimensions ?? [], - aggFields = this.fields.filter(it => it.aggregator), + const dimensions = query.dimensions ?? [], + aggFields = fields.filter(it => it.aggregator), appliedDimNames = dimensions.map( (v, idx) => new Set(dimensions.slice(0, idx + 1).map(it => it.name)) ); @@ -418,7 +419,7 @@ export class View updatedRowDatas.forEach(rowData => this.assignDigest(rowData)); - // Level-derived values move with their inputs, with no leaf-level diff to report them. + // Level-derived values have no stored value to diff - report one changed whenever an input is. // Repeat until a pass adds nothing - derived fields may depend on other derived fields. if (changedFields.size) { for (let added = true; added;) { diff --git a/data/cube/impl/RowDataGenerator.ts b/data/cube/impl/RowDataGenerator.ts index 8c7e8f43e9..cce29324ad 100644 --- a/data/cube/impl/RowDataGenerator.ts +++ b/data/cube/impl/RowDataGenerator.ts @@ -7,7 +7,6 @@ import {PlainObject} from '@xh/hoist/core'; import {isEmpty, isEqual} from 'lodash'; -import type {CubeField} from '../CubeField'; import type {View} from '../View'; import {ViewRowData} from '../ViewRowData'; @@ -92,11 +91,16 @@ export class RowDataGenerator { cubeRowDigest: null, _cubeLeafChildren: null }; - this.view.fields.forEach(field => { - if (field.isDerived && !field.aggregator) { - this.addDerivedGetter(field, proto); + this.view.fields.forEach(({name, isDerived, aggregator, derivedFn}) => { + if (isDerived && !aggregator) { + Object.defineProperty(proto, name, { + get(this: ViewRowData) { + return derivedFn(this); + }, + enumerable: true + }); } else { - data[field.name] = null; + data[name] = null; } }); @@ -106,15 +110,6 @@ export class RowDataGenerator { }; } - private addDerivedGetter({name, derivedFn}: CubeField, target: PlainObject) { - Object.defineProperty(target, name, { - get(this: ViewRowData) { - return derivedFn(this); - }, - enumerable: true - }); - } - private buildLeafClass(): LeafDataClass { if (!this.exposesLeaves) return null; From dfb5eca46e83d08b6b63d67488d0d5c9684b2fab Mon Sep 17 00:00:00 2001 From: lbwexler Date: Fri, 2 Oct 2026 11:09:05 -0400 Subject: [PATCH 4/5] Derived fields: enforce read-only, guard canAggregateFn, doc carry-overs (#4748 items 1, 3, 4) - modifyRecords() throws on a write to a derived field; columns bound to one are never editable. - CubeField throws when a level-derived field sets canAggregateFn. - READMEs: read-by-name, pure/stable-return guidance; replace WeightedAverageAggregator example with the SUM/SUM/ratio decomposition. --- cmp/grid/columns/Column.ts | 4 +- data/Field.ts | 7 ++-- data/README.md | 11 +++++- data/Store.ts | 6 +++ data/cube/CubeField.ts | 4 ++ data/cube/README.md | 76 ++++++++++++-------------------------- 6 files changed, 49 insertions(+), 59 deletions(-) diff --git a/cmp/grid/columns/Column.ts b/cmp/grid/columns/Column.ts index c91020a746..f6703de6c3 100644 --- a/cmp/grid/columns/Column.ts +++ b/cmp/grid/columns/Column.ts @@ -771,8 +771,8 @@ export class Column { /** Does column support editing its field for the given StoreRecord? */ isEditableForRecord(record: StoreRecord): boolean { - const {editable, gridModel} = this; - if (!record) return false; + const {editable, gridModel, field} = this; + if (!record || record.store.getField(field)?.isDerived) return false; return isFunction(editable) ? editable({record, store: record.store, gridModel, column: this}) : editable; diff --git a/data/Field.ts b/data/Field.ts index 1e4047c22d..74b6a05541 100644 --- a/data/Field.ts +++ b/data/Field.ts @@ -74,9 +74,10 @@ export interface FieldSpec { * {name: 'marketValue', dependsOn: ['quantity', 'price'], derivedFn: d => d.quantity * d.price} * ``` * - * Requires `dependsOn`. Derived fields are read-only - {@link Store.modifyRecords} ignores - * writes to them. A Store that is a `projectionOnly` view of another's data (e.g. one - * connected to a Cube View) adopts derived values from its provider rather than computing them. + * Requires `dependsOn`. Derived fields are read-only - {@link Store.modifyRecords} throws on a + * write to one, and a grid column bound to one is never editable. A Store that is a + * `projectionOnly` view of another's data (e.g. one connected to a Cube View) adopts derived + * values from its provider rather than computing them. * * On a {@link CubeField}, the function also runs on every View row where the field is not * aggregated - so a field with an `aggregator` derives at the leaves and rolls up (market diff --git a/data/README.md b/data/README.md index c7874aaaa1..358b1e5f66 100644 --- a/data/README.md +++ b/data/README.md @@ -438,12 +438,21 @@ apps with large datasets. Set `enableXssProtection` per field, or app-wide via A field with a `derivedFn` computes its value from the record's other values, named in the required `dependsOn`. Values are read through a getter on record `data` - never loaded, parsed or written - so they are always current with their inputs, and sort, filter and export like any other -field. Writes to one via `modifyRecords()` are ignored. +field. ```typescript {name: 'marketValue', dependsOn: ['quantity', 'price'], derivedFn: d => d.quantity * d.price} ``` +* **Read-only.** `modifyRecords()` throws on a write to a derived field, and a grid column bound + to one is never editable. +* **Read by name, never enumerated.** The getters are not own properties, so `Object.keys()`, + spread and `JSON.stringify()` omit them. `StoreRecord.getValues()` returns every field, derived + included. +* **Keep the function pure and fast.** It can run once per visible cell per paint and once per + comparison when sorting. Return primitives or stable references - a fresh object or array per + read defeats the equality check grids use to skip repainting unchanged cells. + A `projectionOnly` store adopts derived values from its provider rather than computing them. See the [Cube README](cube/README.md#derived-fields) for derived `CubeField`s, which also run on aggregated View rows. diff --git a/data/Store.ts b/data/Store.ts index 0dfc83a660..7630fd0214 100644 --- a/data/Store.ts +++ b/data/Store.ts @@ -875,6 +875,12 @@ export class Store return; } + const derived = Object.keys(mod).find(it => this._fieldMap.get(it)?.isDerived); + throwIf( + derived, + `Field '${derived}' is derived and read-only - its value is computed at read time and cannot be modified.` + ); + const currentRec = this.getOrThrow(id), updatedData = this.parseUpdate(currentRec.data, mod); diff --git a/data/cube/CubeField.ts b/data/cube/CubeField.ts index 416eb4656c..80cd6cd525 100755 --- a/data/cube/CubeField.ts +++ b/data/cube/CubeField.ts @@ -134,6 +134,10 @@ export class CubeField extends Field { this.isDerived && this.isDimension, `CubeField '${this.name}' may not be both derived and a dimension.` ); + throwIf( + this.isDerived && !this.aggregator && this.canAggregateFn, + `CubeField '${this.name}' is derived at every level and never aggregated - 'canAggregateFn' does not apply.` + ); } //------------------------ diff --git a/data/cube/README.md b/data/cube/README.md index 1b12ec77f6..70a31c4352 100644 --- a/data/cube/README.md +++ b/data/cube/README.md @@ -89,54 +89,20 @@ Extend `Aggregator` and implement `aggregate()` to add application-specific aggr arrive as the row's direct children - a mix of leaf rows and already-aggregated parent rows, typed as `ViewRow` - so most aggregations compose naturally from `row.data[fieldName]`. -Aggregations that cannot be derived from their children's published values alone (a weighted -average, a standard deviation) can keep the extra terms they need as **aggregator state**, via -`AggregationContext.setAggState()` / `getAggState()`. This keeps each row's work proportional to -its child count rather than to its entire subtree of leaves: +Before writing one, check whether the aggregate decomposes into sums. A weighted average is +SUM(price × qty) / SUM(qty) - two derived fields, no aggregator code, and fully incremental: ```typescript -export class WeightedAverageAggregator extends Aggregator { - readonly weightField: string; - - constructor(weightField: string) { - super(); - this.weightField = weightField; - } - - // Reads a second field, so a change to the weight alone must recompute this aggregate - see - // below. This forgoes incremental updates, but not the compositional win of the state below. - override get dependsOnChildrenOnly() { - return false; - } - - override aggregate(rows, fieldName, context) { - let weighted = 0, - weight = 0; - - for (const row of rows) { - // Parents publish an average - compose from their state instead. A parent without - // state did not aggregate this field, so read its published values as for a leaf. - const state = row.isLeaf ? null : context.getAggState(row); - if (state) { - weighted += state.weighted; - weight += state.weight; - } else { - const val = row.data[fieldName], - w = row.data[this.weightField]; - if (val != null && w != null) { - weighted += val * w; - weight += w; - } - } - } - - context.setAggState({weighted, weight}); - return weight ? weighted / weight : null; - } -} +{name: 'weightedPrice', aggregator: 'SUM', dependsOn: ['price', 'qty'], derivedFn: d => d.price * d.qty}, +{name: 'vwap', dependsOn: ['weightedPrice', 'qty'], derivedFn: d => d.weightedPrice / d.qty} ``` -Then reference it from a field: `{name: 'price', aggregator: new WeightedAverageAggregator('qty')}`. +See [Derived Fields](#derived-fields) below. + +Aggregations that do not decompose this way (a standard deviation, a distinct count) can keep the +extra terms they need as **aggregator state**, via `AggregationContext.setAggState()` / +`getAggState()`. This keeps each row's work proportional to its child count rather than to its +entire subtree of leaves. The rows handed to an aggregator are typed as `ViewRow` - the row-level API shared by aggregators and the `lockFn` / `omitFn` / `bucketSpecFn` hooks. Leaf rows additionally carry their source @@ -151,19 +117,19 @@ Rules to observe: * **Expect non-leaf children without state.** `getAggState()` returns null for a child that did not aggregate the field - because its `canAggregateFn` returned false, or because the field is a dimension at that child's level and so is never aggregated there. Such a child publishes a value - to read instead - null in the first case, the dimension value in the second - so treat it as the - example does, exactly like a leaf. + to read instead - null in the first case, the dimension value in the second - so read it exactly + as for a leaf. * **Override `replace()` only if you can keep state consistent** with the value you return. The inherited implementation re-aggregates from direct children, which is correct and already cheap; see `AverageAggregator` for an override that adjusts state from a single leaf's change instead. * **Override `dependsOnChildrenOnly` to return false if the aggregate reads any field other than - its own**, as the weighted average above reads `qty`. A View whose aggregators all depend on - their children only applies a record update incrementally, re-aggregating a field up the - ancestor chain only when that field's own value changed on the leaf - a change to `qty` alone - would leave the weighted `price` stale until the next full rebuild. Returning false routes every - update through a full rebuild, on which reused rows recompute the aggregate afresh. Aggregators - that depend on values beyond their own children (e.g. percent-of-total) must return false for the - same reason, and doing so also gives them access to `AggregationContext.filteredRecords`. + its own.** A View whose aggregators all depend on their children only applies a record update + incrementally, re-aggregating a field up the ancestor chain only when that field's own value + changed on the leaf - a change to another input would leave the aggregate stale until the next + full rebuild. Returning false routes every update through a full rebuild, on which reused rows + recompute the aggregate afresh. Aggregators that depend on values beyond their own children (e.g. + percent-of-total) must return false for the same reason, and doing so also gives them access to + `AggregationContext.filteredRecords`. ## Derived Fields @@ -187,6 +153,10 @@ A derived field with an aggregator is an ordinary measure whose leaf values are than loaded. One without an aggregator is read through a getter on each parent row, so it is always current with that row's aggregates. Derived fields may read other derived fields. +As at the Store layer, values are read by name, never enumerated, and the function should be pure, +fast, and return primitives or stable references - see +[Derived Fields](../README.md#derived-fields) in the data README. + Stores connected to a View adopt these values from the rows they receive and compute nothing themselves - connected stores are always `projectionOnly`. From 64c81b3c9af39be93d5bf118925d52fcebf5cde6 Mon Sep 17 00:00:00 2001 From: lbwexler Date: Fri, 2 Oct 2026 14:08:38 -0400 Subject: [PATCH 5/5] Expand Store changedFields over derived dependents (#4748 item 2) - Store.updateData() adds every derived field reading a named input to the changedFields hint, so grids sorted on a derived column re-sort. - Shared withDerivedDependents() util in data/impl; View uses it instead of its own copy. --- data/Store.ts | 7 ++++++- data/cube/View.ts | 29 +++++------------------------ data/impl/DerivedFields.ts | 31 +++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 25 deletions(-) create mode 100644 data/impl/DerivedFields.ts diff --git a/data/Store.ts b/data/Store.ts index 7630fd0214..2c4a303165 100644 --- a/data/Store.ts +++ b/data/Store.ts @@ -23,6 +23,7 @@ import { StoreValidationResultsMap, ValidationResult } from '@xh/hoist/data'; +import {withDerivedDependents} from '@xh/hoist/data/impl/DerivedFields'; import {StoreValidator} from '@xh/hoist/data/impl/StoreValidator'; import {action, computed, observable, runInAction, observableRef} from '@xh/hoist/mobx'; import {throwIf, warnIf} from '@xh/hoist/utils/js'; @@ -419,6 +420,7 @@ export class Store _created = Date.now(); private _fieldMap: Map; + private _derivedFields: Field[]; experimental: any; /** @internal */ @@ -472,6 +474,7 @@ export class Store this.validator = new StoreValidator({store: this}); this._fieldMap = this.createFieldMap(); + this._derivedFields = this.fields.filter(it => it.isDerived); this._simpleProto = this.createSimpleProto(); this._denseTemplate = this.createDenseTemplate(); this._denseThreshold = this.experimental.denseRecordThreshold ?? DENSE_RECORD_THRESHOLD; @@ -658,8 +661,10 @@ export class Store rawTransaction = rawData; } - const {update, add, remove, rawSummaryData, changedFields, ...other} = rawTransaction; + let {update, add, remove, rawSummaryData, changedFields, ...other} = rawTransaction; throwIf(!isEmpty(other), 'Unknown argument(s) passed to updateData().'); + if (changedFields) + changedFields = withDerivedDependents(changedFields, this._derivedFields); // 1) Pre-process updates and adds into Records let updateRecs: StoreRecord[], addRecs: Map; diff --git a/data/cube/View.ts b/data/cube/View.ts index 64d89df2f3..e9e9ee8ba8 100755 --- a/data/cube/View.ts +++ b/data/cube/View.ts @@ -32,6 +32,7 @@ import {RowDataGenerator} from './impl/RowDataGenerator'; import {BaseRow} from './row/BaseRow'; import {ExposedLeafRow, HiddenLeafRow, LeafRow, LeafUpdateChanges} from './row/LeafRow'; import {AggregateRow, BucketRow} from './row/ParentRow'; +import {withDerivedDependents} from '../impl/DerivedFields'; import {RecordSet, RecordSetDelta} from '../impl/RecordSet'; /** @@ -411,7 +412,9 @@ export class View const {_leafMap, stores, fields} = this, // A producer's changedFields names stored fields only - also check the derived fields // reading them, whose leaf values may have moved without being named. - checkNames = changedFields ? this.withDerivedDependents(changedFields) : null, + checkNames = changedFields + ? withDerivedDependents(changedFields, this._derivedFields) + : null, checkFields = checkNames ? fields.filter(it => checkNames.has(it.name)) : fields, changed: LeafUpdateChanges = {rows: new Set(), fields: new Set()}; @@ -424,7 +427,7 @@ export class View // Derived values on parent rows are read via getter and never diffed - report them changed // to consumers whenever an input is. - changed.fields = this.withDerivedDependents(changed.fields); + changed.fields = withDerivedDependents(changed.fields, this._derivedFields); this.createAggregationContext(); @@ -439,28 +442,6 @@ export class View this.diagnostics.noteUpdate('dataOnly', start); } - /** - * The given field names plus, transitively, every derived query field reading any of them. - * Returns the input set itself when nothing is added. - */ - private withDerivedDependents(names: Set): Set { - const {_derivedFields} = this; - if (isEmpty(_derivedFields) || !names.size) return names; - - let ret = names; - for (let added = true; added;) { - added = false; - _derivedFields.forEach(({name, dependsOn}) => { - if (!ret.has(name) && dependsOn.some(it => ret.has(it))) { - if (ret === names) ret = new Set(names); - ret.add(name); - added = true; - } - }); - } - return ret; - } - // Rows left untouched, but deciding that meant testing the changes against the query. private dataUnchangedUpdate(start: number) { this.info = this.cube.info; diff --git a/data/impl/DerivedFields.ts b/data/impl/DerivedFields.ts new file mode 100644 index 0000000000..943fa3fcb0 --- /dev/null +++ b/data/impl/DerivedFields.ts @@ -0,0 +1,31 @@ +/* + * This file belongs to Hoist, an application development toolkit + * developed by Extremely Heavy Industries (www.xh.io | info@xh.io) + * + * Copyright © 2026 Extremely Heavy Industries Inc. + */ +import type {Field} from '../Field'; +import {isEmpty} from 'lodash'; + +/** + * The given field names plus, transitively, every derived field reading any of them - a change + * to an input is a change to the fields derived from it. Returns the input set itself when + * nothing is added. + * @internal + */ +export function withDerivedDependents(names: Set, derivedFields: Field[]): Set { + if (isEmpty(derivedFields) || !names.size) return names; + + let ret = names; + for (let added = true; added;) { + added = false; + derivedFields.forEach(({name, dependsOn}) => { + if (!ret.has(name) && dependsOn.some(it => ret.has(it))) { + if (ret === names) ret = new Set(names); + ret.add(name); + added = true; + } + }); + } + return ret; +}