diff --git a/CHANGELOG.md b/CHANGELOG.md index 030a20c2a7..70813cabd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,38 @@ 3. Plain ASCII punctuation only. Use " - " for in-sentence breaks, never an em dash. --> +## 89.0.0-SNAPSHOT - unreleased + +### 💥 Breaking Changes (upgrade difficulty: 🟢 LOW - connected stores) + +See [`docs/upgrade-notes/v89-upgrade-notes.md`](docs/upgrade-notes/v89-upgrade-notes.md) for +detailed, step-by-step upgrade instructions with before/after code examples. + +* Stores now connect to a Cube `View` at construction via `StoreConfig.view`, replacing + `ViewConfig.stores` and `View.setStores()` (see `View.addStore()`/`removeStore()` for transient + detach). Connected stores are built as `projectionOnly` projections adopting view rows by + reference - conflicting config (`projectionOnly: false`, `processRawData`, `digestSpec`, + `idEncodesTreePath`) throws. Route edits through the Cube (e.g. `Cube.modifyRecordsAsync()`). +* A connected store's `fields` are now reconciled to its View's query fields at connection and on + query changes - view-published data is described by the query's own `CubeField`s, superseding + any same-named app or grid-inferred field, with app-declared extras preserved. Field metadata + read off the store (types, `displayName`s, calculated status) now flows from the Cube. A + store-layer `calculatedFn` field sharing a view field's name throws at connection. + +### 🎁 New Features + +* Added `FieldSpec.calculatedFn` - declare Store fields computed on the client from each record's + other values and the Store, with no source data or server round-trip required. Values are + computed lazily on read - always current, with minimal memory and load-time overhead - and + support sorting, filtering, and export like any other field. Grids repaint calculated columns + automatically as data changes, and calculated fields are read-only for editing. +* Added `CubeFieldSpec.calculatedFn` - the Cube-layer form of the same concept, computed on View + rows with the View's `AggregationContext`. Recommended for globally-dependent values such as + percent-of-total or ratios of sums such as a weighted average, where a custom aggregator would + slow updates to the entire View - calculated fields keep Views on their fastest incremental + update path. `AggregationContext.filteredRecords` + is readable from these functions and always current. + ## 88.0.0 - 2026-09-28 ### 💥 Breaking Changes (upgrade difficulty: 🔴 HIGH - TC39 decorators + Rsbuild, AG Grid 36, MobX 7, removals) diff --git a/cmp/grid/Grid.ts b/cmp/grid/Grid.ts index ad4f2c8b27..cfd911742d 100644 --- a/cmp/grid/Grid.ts +++ b/cmp/grid/Grid.ts @@ -715,8 +715,9 @@ export class GridLocalModel extends HoistModel { transaction = newRs.diffFrom(prevRs); model.diagnostics.noteGenTransaction(transaction, newRs, prevRs, start); - const applyStart = performance.now(); - if (!this.transactionIsEmpty(transaction)) { + const applyStart = performance.now(), + isEmptyTxn = this.transactionIsEmpty(transaction); + if (!isEmptyTxn) { this.transactionMgr.apply(transaction, prevRs, newRs); } else if (!prevRs) { // AG Grid needs rowData (even if empty) to exit its initial loading state. @@ -743,6 +744,26 @@ export class GridLocalModel extends HoistModel { } } + const calcNames = store.calculatedFieldNames; + if (calcNames.size) { + // Calculated values can move via inputs outside transacted rows - repaint their columns. + const columns = model + .getVisibleLeafColumns() + .filter(c => calcNames.has(c.field)) + .map(c => c.colId); + if (!isEmpty(columns)) agApi.refreshCells({columns}); + + // An empty sync (e.g. summary-only) could still require calc cols resort. + if ( + isEmptyTxn && + prevRs && + !model.externalSort && + model.sortBy.some(s => calcNames.has(model.getColumn(s.colId)?.field)) + ) { + this.transactionMgr.noteSortStale(); + } + } + if (!isEmpty(transaction.add) || !isEmpty(transaction.remove)) { wait().then(() => this.syncSelection()); } diff --git a/cmp/grid/columns/Column.ts b/cmp/grid/columns/Column.ts index c91020a746..8b01e68233 100644 --- a/cmp/grid/columns/Column.ts +++ b/cmp/grid/columns/Column.ts @@ -383,7 +383,9 @@ export interface ColumnSpec { /** * True to make cells in this column editable, or a function to determine on a - * record-by-record basis. + * record-by-record basis. Ignored for columns bound to a calculated field + * ({@link FieldSpec.calculatedFn}) - such values are computed at read time and never + * editable. */ editable?: boolean | ColumnEditableFn; @@ -773,6 +775,7 @@ export class Column { isEditableForRecord(record: StoreRecord): boolean { const {editable, gridModel} = this; if (!record) return false; + if (record.store.calculatedFieldNames.has(this.field)) return false; return isFunction(editable) ? editable({record, store: record.store, gridModel, column: this}) : editable; diff --git a/cmp/grid/impl/GridTransactionManager.ts b/cmp/grid/impl/GridTransactionManager.ts index c355b5e385..eaadb01b1d 100644 --- a/cmp/grid/impl/GridTransactionManager.ts +++ b/cmp/grid/impl/GridTransactionManager.ts @@ -47,8 +47,9 @@ export class GridTransactionManager extends HoistBase { @managed private sortScheduler: DeferredWorkScheduler; - // Rows updated by suppressed transactions since the last sort - null when order is current. - private pendingSortIds: Set = null; + // Rows updated by suppressed transactions since the last sort, or 'full' when order may be + // stale beyond any tracked rows - null when order is current. + private pendingSort: Set | 'full' = null; // Cached provable sort paths - undefined = stale, null = sort not provably value-based. private _sortPaths: Array> | null | undefined; @@ -79,7 +80,7 @@ export class GridTransactionManager extends HoistBase { }); try { agApi.applyTransaction(transaction); - if (!suppress) this.pendingSortIds = null; + if (!suppress) this.pendingSort = null; } finally { agApi.updateGridOptions({ suppressModelUpdateAfterUpdateTransaction: false, @@ -88,6 +89,15 @@ export class GridTransactionManager extends HoistBase { } } + /** + * Note current row order may be stale with no transaction to prove otherwise - e.g. a + * calculated sort value moved by a summary-only update. Schedules a paced full re-sort. + */ + noteSortStale() { + this.pendingSort = 'full'; + this.sortScheduler.scheduleAsync(); + } + //------------------------ // Implementation //------------------------ @@ -119,7 +129,7 @@ export class GridTransactionManager extends HoistBase { // With a flush pending, current order is stale - a delta merge would preserve the // staleness, so any refresh must be full (which resolves the pending flush, per apply). - if (this.pendingSortIds) return 'full'; + if (this.pendingSort) return 'full'; const changedCount = update.length + add.length + remove.length; return newRs.count > 0 && changedCount / newRs.count < this.deltaSortRatio @@ -136,6 +146,12 @@ export class GridTransactionManager extends HoistBase { const sortPaths = this.getSortPaths(); if (!sortPaths) return false; + // Calculated sort values can move via inputs outside any updated row - nothing provable. + const calcNames = this.model.store.calculatedFieldNames; + if (calcNames.size && sortPaths.some(p => calcNames.has(isArray(p) ? p[0] : p))) { + return false; + } + if (changedFields) { return sortPaths.every(p => !changedFields.has(isArray(p) ? p[0] : p)); } @@ -191,25 +207,31 @@ export class GridTransactionManager extends HoistBase { } private notePendingSort(updates: StoreRecord[]) { - const ids = (this.pendingSortIds ??= new Set()); - updates.forEach(rec => ids.add(rec.id)); + const {pendingSort} = this; + // A pending full sort already covers these rows - no need to track them. + if (pendingSort !== 'full') { + const ids = pendingSort ?? (this.pendingSort = new Set()); + updates.forEach(rec => ids.add(rec.id)); + } this.sortScheduler.scheduleAsync(); } private flushPendingSort() { - const {model, pendingSortIds} = this, + const {model, pendingSort} = this, latestRs = model._syncedRs; - // Not ready - leave ids pending; the next transaction will run 'full' and resolve them. - if (!pendingSortIds || !latestRs || !model.isReady) return; - this.pendingSortIds = null; + // Not ready - leave the sort pending; the next transaction will run 'full' and resolve it. + if (!pendingSort || !latestRs || !model.isReady) return; + this.pendingSort = null; const start = performance.now(), {agApi} = model, update = []; - pendingSortIds.forEach(id => { - const rec = latestRs.getById(id); - if (rec) update.push(rec); - }); + if (pendingSort !== 'full') { + pendingSort.forEach(id => { + const rec = latestRs.getById(id); + if (rec) update.push(rec); + }); + } if (update.length && update.length / latestRs.count < this.deltaSortRatio) { agApi.updateGridOptions({deltaSort: true}); diff --git a/data/Field.ts b/data/Field.ts index 60db1469db..b696265ee9 100644 --- a/data/Field.ts +++ b/data/Field.ts @@ -5,7 +5,7 @@ * Copyright © 2026 Extremely Heavy Industries Inc. */ -import {XH} from '@xh/hoist/core'; +import {PlainObject, XH} from '@xh/hoist/core'; import {RuleLike} from '@xh/hoist/data/validation/Types'; import {isLocalDate, LocalDate} from '@xh/hoist/utils/datetime'; import {withDefault} from '@xh/hoist/utils/js'; @@ -13,6 +13,8 @@ import {Rule} from './validation/Rule'; import equal from 'fast-deep-equal'; import {isDate, isString, toNumber, isFinite, startCase, isFunction, castArray} from 'lodash'; import DOMPurify from 'dompurify'; +import type {Store} from './Store'; +import type {CubeCalculatedFn} from './cube/CubeField'; /** * Constructor arguments for a Hoist data package Field. @@ -64,8 +66,51 @@ export interface FieldSpec { * building secured internal apps with large datasets and tight performance tolerances. */ enableXssProtection?: boolean; + + /** + * Function computing this field's value at read time from the record's other values and the + * Store, making this a *calculated* field - derived on the client rather than loaded: + * + * ```ts + * { + * name: 'pctCommission', + * calculatedFn: (data, store) => + * (data.commission / store.summaryRecords[0]?.data.commission) * 100 + * } + * ``` + * + * Values are read via lazy prototype getters on record `data` - never stored, parsed, or + * compared for record reuse, and always current, even when inputs live outside the record + * (e.g. a summary denominator). Grids repaint calculated columns after each transaction, + * and a `FieldFilter` on one triggers a full re-filter. (`FunctionFilter`s are opaque and + * may need a manual {@link Store.refreshFilter}.) + * + * Calculated fields are read-only ({@link Store.modifyRecords} throws, columns are never + * editable, `type` is display-only) and invisible to own-property enumeration - read values + * by name, or via {@link StoreRecord.getValues}. Keep the fn pure and fast (it runs per + * cell paint and per sort comparison), return primitives or stable references, and avoid + * cycles when reading other calculated fields. + * + * See {@link CubeFieldSpec.calculatedFn} for the Cube View form - the union type keeps + * `CubeFieldSpec` assignable wherever `FieldSpec` is accepted; on a plain Store, always + * supply the {@link StoreCalculatedFn} form. + */ + calculatedFn?: StoreCalculatedFn | CubeCalculatedFn; } +/** + * Function computing a Store-level calculated field value at read time. + * See {@link FieldSpec.calculatedFn}. + */ +export type StoreCalculatedFn = (data: PlainObject, store: Store) => any; + +/** + * Function computing a calculated field value at read time - the union of the layer-specific + * signatures declared by {@link FieldSpec.calculatedFn} (Store) and `CubeFieldSpec.calculatedFn` + * (Cube View). + */ +export type CalculatedFn = (data: any, context: any) => any; + /** * Metadata for an individual data field within a {@link StoreRecord}. * @@ -76,6 +121,11 @@ export class Field { return true; } + /** True for {@link CubeField} instances - see that subclass. */ + get isCubeField() { + return false; + } + readonly name: string; readonly type: FieldType; readonly displayName: string; @@ -85,6 +135,19 @@ export class Field { readonly rules: Rule[]; readonly enableXssProtection: boolean; + /** + * Function computing this field's value at read time, marking it as a calculated field. + * Layer-specific signatures - see {@link FieldSpec.calculatedFn} (Store) and + * `CubeFieldSpec.calculatedFn` (Cube View). Not readonly to support subclass assignment + * and anticipated runtime updates to calculated field specs. + */ + calculatedFn: CalculatedFn; + + /** True if this field's value is computed at read time - see {@link FieldSpec.calculatedFn}. */ + get isCalculated(): boolean { + return !!this.calculatedFn; + } + constructor({ name, type = 'auto', @@ -93,7 +156,8 @@ export class Field { defaultValue = null, isDimension = false, rules = [], - enableXssProtection = XH.appSpec.enableXssProtection + enableXssProtection = XH.appSpec.enableXssProtection, + calculatedFn = null }: FieldSpec) { this.name = name; this.type = type; @@ -103,6 +167,7 @@ export class Field { this.rules = this.processRuleSpecs(rules); this.enableXssProtection = enableXssProtection; this.defaultValue = this.parseValueInternal(defaultValue); + this.calculatedFn = calculatedFn; } parseVal(val: any): any { diff --git a/data/README.md b/data/README.md index 813a0fe457..4c14455f7f 100644 --- a/data/README.md +++ b/data/README.md @@ -433,6 +433,51 @@ apps with large datasets. Set `enableXssProtection` per field, or app-wide via | `'tags'` | String array | Splits comma-separated | | `'pwd'` | Password | Marks as sensitive | +### Calculated Fields + +Declare a field with a `calculatedFn` to compute its value on the client at read time, from the +record's other field values and the Store - no source data or server round-trip required: + +```typescript +const store = new Store({ + fields: [ + 'commission', + { + name: 'pctCommission', + calculatedFn: (data, store) => + (data.commission / store.summaryRecords[0]?.data.commission) * 100 + } + ] +}); +``` + +Calculated values are read through lazy prototype getters on record `data` objects - never +stored, parsed, or included in the equality/digest comparisons used to detect unchanged records, +and always current when read. Key characteristics: + +- **Read-only** - grid columns bound to calculated fields are never editable, and + `modifyRecords()` throws on any attempt to write one. `type` is display/metadata only, as + parsing never applies. +- **Works with `projectionOnly`** - record `data` becomes a generated wrapper over the adopted + raw object, adding the calculated getters with no per-record copy of source values. +- **Read by name, never enumerated** - calculated values live behind prototype getters, invisible + to own-property enumeration: `Object.keys()`, spread and `JSON.stringify()` omit them (record + `data` should never be enumerated in any case). `StoreRecord.getValues()` returns a plain-object + copy of all field values, calculated included. +- **Automatic grid repaint** - grids bound to the Store refresh columns displaying calculated + fields after each transaction, repainting visible cells whose value moved via an input outside + their own row (e.g. a summary denominator). +- **Automatic filter refresh** - a `FieldFilter` testing a calculated field triggers a full + re-filter on each transaction, keeping membership current. `FunctionFilter`s are opaque to this + detection - one reading calculated values may require a manual `refreshFilter()`. +- Sorting and exporting read through the getters and work naturally - keep `calculatedFn` a fast, + pure function, as it can run once per visible cell per paint and once per comparison when + sorting. Prefer returning primitives or stable references - a fresh object or array per read + defeats the value-equality check grids use to skip repainting unchanged cells. + +See `CubeFieldSpec.calculatedFn` (`data/cube/README.md`) for the Cube-layer form of the same +concept, computed on View rows with an `AggregationContext`. + ## Filter System **Files**: `filter/Filter.ts`, `filter/FieldFilter.ts`, `filter/CompoundFilter.ts`, `filter/FunctionFilter.ts` @@ -961,21 +1006,17 @@ parses and owns. Store then uses each incoming raw object *as* its record's `dat This collapses the usual two objects per row to one, and skips the per-row parse on every load and update. -Use this config for stores connected to a Cube `View`, or fed by an endpoint that returns data in -its final client-side form. A View logs a warning when a connected store leaves the config unset. -Set it explicitly to `false` to opt out and silence that warning. +Use this config for stores fed by an endpoint that returns data in its final client-side form. +Stores connected to a Cube `View` (constructed with `StoreConfig.view`) are always projections - +the flag is set automatically, and conflicting config throws. ```typescript -const store = new Store({ - fields: [...], - projectionOnly: true -}); - const view = cube.createView({ query: {dimensions: ['region', 'product']}, - stores: store, connect: true }); + +const store = new Store({view, fields: [...]}); ``` This mode carries real constraints: diff --git a/data/Store.ts b/data/Store.ts index ce3c46f754..344d0fd212 100644 --- a/data/Store.ts +++ b/data/Store.ts @@ -14,6 +14,7 @@ import { FilterBindTarget, FilterLike, FilterValueSource, + flattenFilter, parseFilter, StoreRecord, StoreRecordDigest, @@ -40,13 +41,16 @@ import { isNil, isNull, isString, + map, partition, remove as lodashRemove, uniq, uniqBy, values } from 'lodash'; +import type {View} from './cube/View'; import {instanceManager} from '../core/impl/InstanceManager'; +import {RecordDataGenerator} from './impl/RecordDataGenerator'; import {RecordSet} from './impl/RecordSet'; import {StoreDiagnostics} from './impl/StoreDiagnostics'; @@ -196,10 +200,8 @@ export interface StoreConfig { /** * True to mark this store as a read-only projection of data owned and parsed elsewhere. - * Recommended for stores connected to a Cube {@link View} for improved performance, when no - * additional record parsing or local data modification is required. Default null - a View - * logs a warning when its connected stores leave this unset. Set explicitly to `false` to - * opt out and silence the warning. + * Default null. Stores connected to a Cube {@link View} (via {@link StoreConfig.view}) are + * always projections - the flag is set automatically at construction. * * Each incoming raw object is used *as* its record's `data`, by reference, skipping the * per-record parse and copy on every load and update. Raw data must already match what the @@ -219,6 +221,15 @@ export interface StoreConfig { */ projectionOnly?: boolean; + /** + * Cube {@link View} to connect this store to, fixed for the store's lifetime. The store is + * built as a read-only projection of the View's published rows: `projectionOnly` and a + * row-based `digestSpec` are set automatically, the View's query fields are merged into + * `fields` (view fields win on name collisions; app-declared extras are preserved), and the + * View registers and loads this store. Conflicting config throws. + */ + view?: View; + /** * Set to true to always validate all uncommitted records on every change to * uncommitted records (add, modify, or remove). Default false. @@ -372,8 +383,10 @@ export class Store idEncodesTreePath: boolean; freezeData: boolean; retainRaw: boolean; - readonly projectionOnly: boolean; + projectionOnly: boolean; validationIsComplex: boolean; + /** Connected Cube View, set at construction - see {@link StoreConfig.view}. */ + readonly view: View = null; @observableRef accessor filter: Filter; @@ -403,12 +416,18 @@ export class Store @observableRef private accessor _current: RecordSet; @observableRef accessor _filtered: RecordSet; - private _dataTemplate: PlainObject = null; - private _dataDefaults: PlainObject = null; private _denseRecordThreshold: number; private _digestSpec: StoreRecordDigestSpec; private _digestFn: (raw: PlainObject) => StoreRecordDigest; + // Record data generation + calculated field support - see generateDataConfig(). + private _dataGenerator: RecordDataGenerator; + private _calculatedFieldNames: Set = null; + private _filterHasCalcFields: boolean = null; + + // Fields as declared at construction - the app's own set, preserved across view-field merges. + private _ownFields: Field[]; + // Last parent pair verified position-equal by positionUnchanged(). private _verifiedCachedParent: StoreRecord = null; private _verifiedNewParent: StoreRecord = null; @@ -441,6 +460,7 @@ export class Store digestSpec = null, retainRaw = true, projectionOnly = null, + view = null, validationIsComplex = false, experimental, xhName = null, @@ -451,10 +471,27 @@ export class Store projectionOnly && processRawData, 'Store.projectionOnly cannot be used with processRawData - a projection adopts data already parsed by its provider.' ); + if (view) { + throwIf( + projectionOnly === false || + processRawData || + idEncodesTreePath || + (digestSpec != null && digestSpec !== 'cubeRowDigest'), + 'A Store connected to a Cube View is built as a read-only projection of its published rows - remove conflicting `projectionOnly`, `processRawData`, `digestSpec`, or `idEncodesTreePath` config.' + ); + throwIf( + data, + 'A Store connected to a Cube View is loaded by that View - remove the `data` config.' + ); + projectionOnly = true; + digestSpec = 'cubeRowDigest'; + } + this.view = view; this.xhName = xhName; this.experimental = this.parseExperimental(experimental); - this.fields = this.parseFields(fields, fieldDefaults); + this._ownFields = this.parseFields(fields, fieldDefaults); + this.fields = view ? this.mergeViewFields(view.fields) : this._ownFields; this.idSpec = this.parseIdSpec(idSpec); this.processRawData = processRawData; this.filter = parseFilter(filter); @@ -473,14 +510,13 @@ export class Store this.resetRecords(); this.validator = new StoreValidator({store: this}); - this._fieldMap = this.createFieldMap(); - this._dataDefaults = this.createDataDefaults(); - this._dataTemplate = {...this._dataDefaults}; // Clone for fast-props mode. + this.generateDataConfig(); this._denseRecordThreshold = this.experimental.denseRecordThreshold ?? DENSE_RECORD_THRESHOLD; if (data) this.loadData(data); instanceManager.registerStore(this); + view?.addStore(this); } /** See {@link StoreConfig.digestSpec} - settable, taking effect on the next load. */ @@ -552,6 +588,8 @@ export class Store if (updated !== _committed || updated !== _current) { this._committed = this._current = updated; this.incrementalRefilter(); + } else if (this.filterReferencesCalculatedFields()) { + this.fullRefilter(); } this.lastLoaded = this.lastUpdated = Date.now(); @@ -756,7 +794,11 @@ export class Store } this.diagnostics.noteUpdate(this._current, prevCurrent, start); - if (hasChanges) this.incrementalRefilter(); + if (hasChanges) { + this.incrementalRefilter(); + } else if (changeLog.summaryRecords && this.filterReferencesCalculatedFields()) { + this.fullRefilter(); + } if (!isEmpty(changeLog)) { this.lastUpdated = Date.now(); @@ -992,6 +1034,48 @@ export class Store return this.fields.map(it => it.name); } + /** + * Names of all fields on this Store whose values are computed at read time rather than + * loaded - fields declared with {@link FieldSpec.calculatedFn}, including view-published + * calculated `CubeField`s adopted from a connected Cube View. Grids bound to this Store use + * this set to automatically repaint calculated columns after each data transaction. + * @internal + */ + get calculatedFieldNames(): Set { + return this._calculatedFieldNames; + } + + /** @internal - called by the connected View when its query fields change. */ + reconcileFields(viewFields: Field[]) { + const newFields = this.mergeViewFields(viewFields), + unchanged = + newFields.length === this.fields.length && + newFields.every((it, idx) => it === this.fields[idx]); + if (unchanged) return; + + this.fields = newFields; + this.generateDataConfig(); + } + + // Merge view-published fields with the app's own extras - types, displayNames and calculated + // status flow from the query's CubeFields, which supersede app fields sharing their name. + private mergeViewFields(viewFields: Field[]): Field[] { + const viewNames = new Set(map(viewFields, 'name')), + extras = this._ownFields.filter(it => !viewNames.has(it.name)); + + // A store-layer calculatedFn on a view-published name is a genuine conflict - the app fn + // would shadow the view's value, with both claiming to compute the field. + const conflict = this._ownFields.find( + it => viewNames.has(it.name) && it.isCalculated && !it.isCubeField + ); + throwIf( + conflict, + `Store field '${conflict?.name}' declares a calculatedFn but is also published by the connected Cube View - rename the store-layer field, or compute it on the View via CubeFieldSpec.calculatedFn.` + ); + + return [...viewFields, ...extras]; + } + /** * Records in this store, respecting any filter (if applied). * Order is not a guaranteed property of a Store - sort explicitly where order matters. @@ -1088,6 +1172,7 @@ export class Store filter = parseFilter(filter); if (this.filter != filter && !this.filter?.equals(filter)) { this.filter = filter; + this._filterHasCalcFields = null; this.incrementalRefilter(); } @@ -1358,12 +1443,29 @@ export class Store @action private incrementalRefilter() { + // Calculated values can cross a filter threshold via inputs outside any transacted row. + if (this.filterReferencesCalculatedFields()) { + this.fullRefilter(); + return; + } + const start = performance.now(), {_current, _filtered: prevFiltered} = this; this._filtered = _current.withFilter(this.filter, prevFiltered); this.diagnostics.noteFilter(this._filtered, _current, prevFiltered, start); } + // FieldFilters declare their field; FunctionFilters are opaque and may need a manual + // refreshFilter() when external inputs change. Memoized on filter/calc field name changes. + private filterReferencesCalculatedFields(): boolean { + return (this._filterHasCalcFields ??= + !!this.filter && + this.calculatedFieldNames.size > 0 && + flattenFilter(this.filter).some(it => + this.calculatedFieldNames.has((it as any).field) + )); + } + @action private fullRefilter() { const start = performance.now(), @@ -1391,22 +1493,29 @@ export class Store } // 2) Projections adopt raw data with no reparsing. Value identical rows - // can be re-used (instance identical reuse requires a digest above) + // can be re-used (instance identical reuse requires a digest above). Calculated fields + // are excluded from the comparison - computed values carry no signal of their own. if (this.projectionOnly) { const cachedData = cached?.data; if ( cachedData && - raw !== cachedData && - this.fields.every(({name}) => equal(raw[name], cachedData[name])) + raw !== cached.raw && + this._dataGenerator.equalityFields.every(({name}) => + equal(raw[name], cachedData[name]) + ) ) { return cached; } + + // With calculated fields, adopt the raw via a generated wrapper carrying their + // getters (data !== raw) - otherwise adopt the raw object itself, as-is. + const data = this._dataGenerator.projectionData(raw); return new StoreRecord({ id, store: this, raw, - data: raw, - committedData: raw, + data, + committedData: data, parent, isSummary, digest @@ -1519,7 +1628,7 @@ export class Store rescuable = !!cached; for (const name in data) { const field = _fieldMap.get(name); - if (field) { + if (field && !field.isCalculated) { const val = field.parseVal(data[name]); if (val !== field.defaultValue) { if (rescuable) { @@ -1545,8 +1654,15 @@ export class Store hasOwn = Object.prototype.hasOwnProperty; let n = 0; this.fields.forEach(field => { - const {name} = field, - val = hasOwn.call(update, name) ? field.parseVal(update[name]) : data[name]; + const {name} = field; + if (field.isCalculated) { + throwIf( + hasOwn.call(update, name), + `Field '${name}' is calculated and read-only - its value is computed at read time and cannot be modified.` + ); + return; + } + const val = hasOwn.call(update, name) ? field.parseVal(update[name]) : data[name]; if (val !== field.defaultValue) { names[n] = name; vals[n] = val; @@ -1560,27 +1676,18 @@ export class Store } /** - * Build a record `data` object from the non-default entries buffered in `_recordBuildData`, - * choosing its representation by their count: - * - * - Below `denseRecordThreshold`, a sparse object - own properties for the buffered values - * only, defaults reached through the shared `_dataDefaults` prototype. Costs nothing for - * unpopulated fields, and stays safely inside V8's fast-properties mode at these counts. - * - At or above it, a clone of the shared template carrying every Field. Wide objects built - * by per-property adds are demoted to V8's dictionary mode - cloning sidesteps the adds - * (overwriting an existing property is not an add), so all dense records share the - * template's one fixed shape. - * - * The representation is decided per record, from parsed content alone - records with equal - * field values always take equal shapes, which the deep-equal comparisons in modifyRecords() - * require. + * Build a record `data` object from the non-default entries buffered in `_recordBuildData` - + * sparse below `denseRecordThreshold`, dense at or above it (see {@link RecordDataGenerator}). + * Decided per record from parsed content alone, so records with equal field values always + * take equal shapes, as the deep-equal comparisons in modifyRecords() require. */ private buildData(): PlainObject { const {names, vals, n} = this._recordBuildData, + {_dataGenerator} = this, ret = n >= this._denseRecordThreshold - ? {...this._dataTemplate} - : Object.create(this._dataDefaults); + ? _dataGenerator.denseData() + : _dataGenerator.sparseData(); for (let i = 0; i < n; i++) { ret[names[i]] = vals[i]; } @@ -1595,16 +1702,20 @@ export class Store } /** - * Shared template for record `data` objects - an own property for every Field, holding its - * defaultValue. `parseOrRescue()` clones it per record, so all records in a Store share one - * identical, fixed shape. That keeps them in V8's compact fast-properties mode: objects built - * instead by per-field property adds are demoted to a per-object hashtable ("dictionary mode") - * past ~20 adds, costing several times more memory per record. + * (Re)generate the per-Store constructs backing record `data` objects - field map and the + * record data generator. Re-runnable, so field reconciliation and anticipated + * runtime calculated-field updates can regenerate. Existing records are not re-wrapped. */ - private createDataDefaults() { - const ret = {}; - this.fields.forEach(({name, defaultValue}) => (ret[name] = defaultValue)); - return ret; + private generateDataConfig() { + this._fieldMap = this.createFieldMap(); + this._dataGenerator = new RecordDataGenerator(this); + this._calculatedFieldNames = new Set( + map( + this.fields.filter(it => it.isCalculated), + 'name' + ) + ); + this._filterHasCalcFields = null; } private createFieldMap() { diff --git a/data/StoreRecord.ts b/data/StoreRecord.ts index 9c6f097f58..ae89abbf8a 100644 --- a/data/StoreRecord.ts +++ b/data/StoreRecord.ts @@ -50,7 +50,11 @@ export class StoreRecord { * enumeration of all field values, or {@link getModifiedValues} for locally-modified values * only. * - * With {@link StoreConfig.projectionOnly}, this is the raw source object itself. + * With {@link StoreConfig.projectionOnly}, this is the raw source object itself - or, when + * the Store declares calculated fields, a generated wrapper reading through it. + * + * Values of calculated fields ({@link FieldSpec.calculatedFn}) are computed lazily by + * prototype getters when read from this object - they are never stored. */ readonly data: PlainObject; @@ -248,9 +252,11 @@ export class StoreRecord { const {data, committedData} = this, ret: PlainObject = {}; - this.fields.forEach(({name}) => { - const val = data[name]; - if (!equal(val, committedData[name])) ret[name] = val; + this.fields.forEach(({name, isCalculated}) => { + if (!isCalculated) { + const val = data[name]; + if (!equal(val, committedData[name])) ret[name] = val; + } }); if (!isEmpty(ret)) { ret.id = this.id; diff --git a/data/cube/Cube.ts b/data/cube/Cube.ts index ba0c9d1e2a..644c13f100 100755 --- a/data/cube/Cube.ts +++ b/data/cube/Cube.ts @@ -271,30 +271,28 @@ export class Cube extends HoistBase { * returned by {@link Cube.executeQuery}, a View created with this method can be configured * with `connect:true` to automatically update as the underlying data in the Cube changes. * - * Provide one or more `stores` to automatically populate them with the aggregated data returned - * by the query, or read the returned {@link View.result} directly. + * Construct Stores against the returned View via {@link StoreConfig.view} to automatically + * populate them with the aggregated data returned by the query, or read the returned + * {@link View.result} directly. * * When the returned View is no longer needed, call {@link View.destroy} (or save a reference * via an `@managed` model property) to avoid unnecessary processing. * * @param query - query to be used to construct this view. - * @param stores - Stores to be automatically loaded/reloaded with View results. * @param connect - true to update View automatically when data in the underlying Cube changes. + * @param xhName - see {@link HoistBase.xhName}. */ createView({ query, - stores, connect = false, xhName = null }: { query: QueryConfig; - stores?: Store[] | Store; connect?: boolean; xhName?: string; }): View { return new View({ query: new Query({...query, cube: this}), - stores, connect, xhName }); diff --git a/data/cube/CubeField.ts b/data/cube/CubeField.ts index 0fd5fa177c..f6cbd54c61 100755 --- a/data/cube/CubeField.ts +++ b/data/cube/CubeField.ts @@ -23,7 +23,9 @@ import { SumStrictAggregator, UniqueAggregator } from '@xh/hoist/data'; +import {throwIf} from '@xh/hoist/utils/js'; import {isString} from 'lodash'; +import type {ViewRowData} from './ViewRowData'; export interface CubeFieldSpec extends FieldSpec { /** @@ -37,6 +39,40 @@ export interface CubeFieldSpec extends FieldSpec { */ aggregator?: Aggregator | AggregatorToken; + /** + * Function computing this field's value at read time on every View row, from the row's other + * values and the View's {@link AggregationContext} - the Cube-layer form of + * {@link FieldSpec.calculatedFn}, with a widened signature. + * + * Carrying no aggregator, calculated fields never disqualify a View from its incremental + * data-only update path - the recommended way to express globally-dependent values like + * percent-of-total: + * + * ```ts + * { + * name: 'pctCommission', + * calculatedFn: (row, ctx) => { + * const total = sumBy(ctx.filteredRecords, r => r.data.commission); + * return total ? (row.commission / total) * 100 : null; + * } + * } + * ``` + * + * When the needed global is already published as a row (e.g. `includeRoot` + + * `loadRootAsSummary`), prefer a Store-layer `calculatedFn` reading `store.summaryRecords`. + * Otherwise read `ctx.filteredRecords` here, memoizing per-tick intermediates in + * `ctx.appData`. Ratios of aggregates (e.g. a weighted average as `SUM(priceQty) / SUM(qty)`) + * are best expressed as a calculated field over two `SUM` fields - see the Cube README. + * Shared semantics per {@link FieldSpec.calculatedFn}: read-only, read by + * name (never own-property enumeration), and prefer returning primitives or stable + * references. + * + * Mutually exclusive with `aggregator`, `canAggregateFn` and `isDimension`. Calculated + * fields may not feed other aggregators or appear in a {@link BucketSpec}'s + * `dependentFields`. + */ + calculatedFn?: CubeCalculatedFn; + /** * Function to determine if aggregation should be performed at a given level of a query result. * @@ -87,6 +123,12 @@ export type CanAggregateFn = ( context: AggregationContext ) => boolean; +/** + * Function computing a Cube-layer calculated field value at read time. + * See {@link CubeFieldSpec.calculatedFn}. + */ +export type CubeCalculatedFn = (row: ViewRowData, context: AggregationContext) => any; + /** * Metadata used to define a measure or dimension in Cube. For properties present on raw data source * objects to be included in a Cube, the Cube must be configured with a matching Field that tells @@ -100,6 +142,13 @@ export class CubeField extends Field { isLeafDimension: boolean; parentDimension: string; + override get isCubeField() { + return true; + } + + /** See {@link CubeFieldSpec.calculatedFn} - Cube-layer signature. */ + declare calculatedFn: CubeCalculatedFn; + static averageAggregator = new AverageAggregator(); static averageStrictAggregator = new AverageStrictAggregator(); static childCountAggregator = new ChildCountAggregator(); @@ -117,6 +166,7 @@ export class CubeField extends Field { canAggregateFn = null, isLeafDimension = false, parentDimension = null, + calculatedFn = null, ...fieldArgs }: CubeFieldSpec) { super(fieldArgs); @@ -128,6 +178,13 @@ export class CubeField extends Field { // Dimension specific this.isLeafDimension = isLeafDimension; this.parentDimension = parentDimension; + + // Calculated - carries the widened Cube-layer signature, assigned post-super. + this.calculatedFn = calculatedFn; + throwIf( + calculatedFn && (this.aggregator || this.canAggregateFn || this.isDimension), + `CubeField '${this.name}' may not combine 'calculatedFn' with 'aggregator', 'canAggregateFn', or 'isDimension' - calculated values are computed at read time, never aggregated or grouped on.` + ); } //------------------------ diff --git a/data/cube/README.md b/data/cube/README.md index f07f79161a..a7de2ec4cf 100644 --- a/data/cube/README.md +++ b/data/cube/README.md @@ -163,7 +163,71 @@ Rules to observe: would leave the weighted `price` stale until the next full rebuild. Returning false routes every update through a full rebuild, on which reused rows recompute the aggregate afresh. Aggregators that depend on values beyond their own children (e.g. percent-of-total) must return false for the - same reason, and doing so also gives them access to `AggregationContext.filteredRecords`. + same reason, and doing so also gives them access to `AggregationContext.filteredRecords`. Where + the aggregate is a ratio of sums, prefer restating it with a [calculated field](#calculated-fields) + - that keeps the View on the incremental path. + +## Calculated Fields + +Declare a `CubeField` with a `calculatedFn` to compute its value at read time on every View row, +from the row's other values and the View's `AggregationContext` - the Cube-layer form of +`FieldSpec.calculatedFn` (see `data/README.md`), with a widened signature: + +```typescript +{ + name: 'pctCommission', + calculatedFn: (row, ctx) => { + // Memoize per-update intermediates in ctx.appData - the context is replaced whenever + // the record set changes in any way. + const total = (ctx.appData.totalCommission ??= sumBy( + ctx.filteredRecords, + r => r.data.commission + )); + return total ? (row.commission / total) * 100 : null; + } +} +``` + +Calculated values are read through lazy prototype getters on the `ViewRowData` objects a View +publishes and are never stored or aggregated. Because they carry no aggregator, they never +disqualify a View from its incremental data-only update path - making them the recommended way to +express globally-dependent values like percent-of-total, in place of an eagerly-computed +aggregator reading beyond its own children. `ctx.filteredRecords` stays readable and fresh on +those incremental updates, rebuilt lazily from the View's leaves (at most one O(n) pass per +update, and only if read). As at the Store layer, values are read by name - never own-property +enumeration - and fns should return primitives or stable references so grid change detection can +skip unchanged cells. + +Note when the needed global is already published as a row - e.g. a View with `includeRoot` +loading a store with `loadRootAsSummary` - prefer a Store-layer `calculatedFn` reading +`store.summaryRecords`, with no Cube API needed at all. + +**Ratios over aggregates - e.g. a weighted average.** An aggregate that reads a second field, such +as a price weighted by quantity, cannot use the incremental update path as a custom aggregator +(see [Custom Aggregators](#custom-aggregators)). Restate it instead as built-in `SUM`s over +leaf-level terms, with a calculated field taking the ratio on every row: + +```typescript +// Leaf records supply `qty` and a precomputed `priceQty` (price * qty) - from the server, or via +// the Cube's `processRawData`. Calculated fields never feed aggregators, so the product must be +// a stored leaf value. +fields: [ + {name: 'qty', aggregator: 'SUM'}, + {name: 'priceQty', aggregator: 'SUM'}, + {name: 'price', calculatedFn: row => (row.qty ? row.priceQty / row.qty : null)} +] +``` + +Both sums update incrementally in O(depth) per changed leaf, whether `price` or `qty` changed, and +the ratio is always current on aggregate and leaf rows alike - no aggregator state, no +`dependsOnChildrenOnly: false`. The same decomposition covers other ratios, conditional sums, and +variance (via sum, sum of squares and `LEAF_COUNT`). Note `Cube.modifyRecordsAsync()` does not +re-run `processRawData`, so a product derived there goes stale on local edits until the next load +or update - supply it from the server if leaf-level editing matters. + +Constraints: `calculatedFn` is mutually exclusive with `aggregator`, `canAggregateFn` and +`isDimension`; calculated fields may not feed other aggregators or appear in a `BucketSpec`'s +`dependentFields`. ## Querying with Views @@ -178,7 +242,6 @@ const view = cube.createView({ dimensions: ['region', 'product'], filter: {field: 'year', op: '=', value: 2024} }, - stores: store, connect: true // Auto-update when cube data changes }); ``` @@ -198,12 +261,11 @@ const view = cube.createView({ dimensions: ['region', 'product'], includeRoot: true }, - stores: new Store({loadRootAsSummary: true}), connect: true }); // The connected GridModel can then show the root as a summary row: -const gridModel = new GridModel({store, showSummary: true, ...}); +const gridModel = new GridModel({store: {view, loadRootAsSummary: true}, showSummary: true, ...}); ``` **Leaf-level drill-down with `includeLeaves`:** @@ -216,7 +278,6 @@ const view = cube.createView({ dimensions: ['region'], includeLeaves: true }, - stores: store, connect: true }); // In a tree grid, expanding "North America" shows its aggregated children, @@ -234,7 +295,6 @@ const view = cube.createView({ dimensions: ['region', 'product'], provideLeaves: true }, - stores: store, connect: true }); ``` @@ -249,7 +309,6 @@ const view = cube.createView({ includeRoot: true, // Single row with grand totals filter: {field: 'region', op: '=', value: 'EMEA'} }, - stores: store, connect: true }); ``` @@ -299,24 +358,35 @@ There are two ways to consume View results: **Option 1: Connected stores (recommended for grids)** -Provide one or more stores via `ViewConfig.stores`. The View auto-loads hierarchical data -into them whenever the query results change. Configure connected stores with -`projectionOnly: true` (adopt View rows as record data without re-parsing). Record reuse is +Construct stores against the View via `StoreConfig.view` - each registers with the View at +construction and is auto-loaded whenever the query results change. Connected stores are always `projectionOnly` +projections - the View sets this itself - adopting View rows as record data without re-parsing. +Record reuse is automatic - the View installs its own row-based digest on each connected store, so rows -republished without change skip record rebuilds: +republished without change skip record rebuilds. -```typescript -const store = new Store({ - fields: [...], - projectionOnly: true -}); +Field metadata flows from the View as well: at construction (and on query changes), the store's +`fields` are reconciled to the query's own `CubeField`s - types, `displayName`s, and calculated +status carry through to everything reading field metadata off the store (filter fields, choosers, +editability). There is no need to redeclare view-published fields on the store - declare only +extras, such as store-layer calculated fields composed over view rows. An app field sharing a +view field's name is superseded by the view's; customize display metadata for view-published +fields on the `CubeField` itself. +```typescript const view = cube.createView({ query: {dimensions: ['region', 'product']}, - stores: store, connect: true }); +const store = new Store({ + view, + fields: [ + // View-published fields adopted automatically - declare only store-layer extras, e.g.: + {name: 'pctOfTotal', calculatedFn: (data, store) => ...} + ] +}); + // Use the store with a GridModel const gridModel = new GridModel({store, treeMode: true, columns: [...]}); ``` diff --git a/data/cube/View.ts b/data/cube/View.ts index d088b300a3..f3dc76c583 100755 --- a/data/cube/View.ts +++ b/data/cube/View.ts @@ -6,7 +6,7 @@ */ import type {GridFilterBindTarget} from '@xh/hoist/cmp/grid'; -import {HoistBase, PlainObject, Some} from '@xh/hoist/core'; +import {HoistBase, PlainObject} from '@xh/hoist/core'; import {instanceManager} from '@xh/hoist/core/impl/InstanceManager'; import { Cube, @@ -25,7 +25,7 @@ import {ViewRowData} from '@xh/hoist/data/cube/ViewRowData'; import {ViewDiagnostics} from './impl/ViewDiagnostics'; import {action, observable, observableRef} from '@xh/hoist/mobx'; import {throwIf} from '@xh/hoist/utils/js'; -import {castArray, forEach, groupBy, isEmpty, isNil, map} from 'lodash'; +import {forEach, groupBy, isEmpty, isNil, map, partition} from 'lodash'; import {AggregationContext} from './aggregate/AggregationContext'; import {RowCache} from './impl/RowCache'; import {RowDataGenerator} from './impl/RowDataGenerator'; @@ -47,16 +47,6 @@ export interface ViewConfig { /** Query to be used to construct this view. */ query: Query; - /** - * Store(s) to be automatically (re)loaded with data from this view. - * Optional - read {@link View.result} directly to use without a Store. - * - * Connected stores should generally set {@link StoreConfig.projectionOnly} - view rows are - * already parsed and owned by this View, so adopting them directly improves performance - * when no additional record parsing or local data modification is required. - */ - stores?: Store[] | Store; - /** * True to reactively update the View's {@link View.result} and any connected store(s) when data * in the underlying Cube changes. False (default) to have this view run its query once to @@ -125,7 +115,8 @@ export class View @observableRef accessor result: ViewResult = null; /** Stores to which results of this view should be (re)loaded. */ - stores: Store[] = null; + /** Connected stores - registered at their construction via {@link StoreConfig.view}. */ + stores: Store[] = []; /** The source {@link Cube.info} as of the last time the view was updated. */ @observableRef accessor info: PlainObject = null; @@ -158,6 +149,8 @@ export class View _aggFieldNamesByDepth: Set[] = null; _canAggregateFnFieldsByDepth: CubeField[][] = null; _complexAggFieldsByDepth: CubeField[][] = null; + _calcFields: CubeField[] = null; + _nonCalcFields: CubeField[] = null; _aggContext: AggregationContext = null; _rowCache: RowCache = null; @@ -166,14 +159,13 @@ export class View super(); const start = performance.now(), - {query, stores = [], connect = false, xhName = null} = config; + {query, connect = false, xhName = null} = config; this.xhName = xhName; this.query = query; - this.stores = this.parseStores(stores); + this.buildIndices(); this._rowCache = new RowCache(this); this._rowDataGenerator = new RowDataGenerator(this); - this.buildIndices(); this.fullUpdate('query', start); if (connect) { @@ -236,8 +228,9 @@ export class View if (oldQuery.equals(newQuery)) return; this.query = newQuery; - this._rowDataGenerator.onQueryChange(); this.buildIndices(); + this._rowDataGenerator.onQueryChange(); + this.stores.forEach(s => s.reconcileFields(this.fields)); // If the cube is changing potentially disconnect from the old cube and connect to the new const {cube: oldCube} = oldQuery, @@ -277,10 +270,24 @@ export class View return this._fieldsByName.get(name); } - /** Set stores to be loaded/reloaded with data from this view. */ - setStores(stores: Some) { - this.stores = this.parseStores(stores); - this.loadStores(); + /** + * Attach a store constructed for this View via {@link StoreConfig.view}, loading it with + * current results - called by the Store's own constructor, and to re-attach a store after + * `removeStore()`. No-op if already attached. + */ + addStore(store: Store) { + throwIf( + store.view !== this, + 'Store was not constructed for this View - connect stores at construction via `StoreConfig.view`.' + ); + if (this.stores.includes(store)) return; + this.stores = [...this.stores, store]; + this.loadStores([store]); + } + + /** Detach a connected store - it retains its configuration and last-loaded data. */ + removeStore(store: Store) { + this.stores = this.stores.filter(it => it !== store); } /** Update the filter on the current Query.*/ @@ -355,6 +362,7 @@ export class View private buildIndices() { this._fieldsByName = new Map(this.fields.map(it => [it.name, it])); + [this._calcFields, this._nonCalcFields] = partition(this.fields, f => f.isCalculated); // Aggregation eligibility is a function of level alone - dimensions apply in order, and // bucket rows share the level of the aggregate row above them. Note depth 0 has no applied @@ -384,7 +392,7 @@ export class View private fullUpdate(trigger: 'load' | 'update' | 'query', start: number) { this.filterRecords(); - this.createAggregationContext(); + this.createAggregationContext(() => this._records.list); this.generateRows(); this.loadStores(); this.updateResults(); @@ -405,8 +413,10 @@ export class View // Apply value changes to leaves already in the view, adjusting ancestor aggregates in place. private dataOnlyUpdate(updates: StoreRecord[], changedFields: Set, start: number) { - const {_leafMap, stores, fields} = this, - checkFields = changedFields ? fields.filter(it => changedFields.has(it.name)) : fields, + const {_leafMap, stores, _nonCalcFields} = this, + checkFields = changedFields + ? _nonCalcFields.filter(it => changedFields.has(it.name)) + : _nonCalcFields, changed: LeafUpdateChanges = {rows: new Set(), fields: new Set()}; // `_records` left stale by design - simple updates never touch filter/dim/bucket fields. @@ -416,7 +426,10 @@ export class View changed.rows.forEach(rowData => this.assignDigest(rowData)); - this.createAggregationContext(); + // If filtered records needed for complex aggregators, or calculated columns re-derive. + this.createAggregationContext(() => + Array.from(this._leafMap.values(), it => it.cubeRecord) + ); stores.forEach(store => { const recordUpdates = []; @@ -436,13 +449,13 @@ export class View this.diagnostics.noteUpdate('unchanged', start); } - private loadStores() { + private loadStores(stores: Store[] = this.stores) { const {_leafMap, _rowDatas} = this; if (!_leafMap || !_rowDatas) return; // Skip degenerate root in stores/grids, but preserve in object api. const storeRows = _leafMap.size !== 0 ? _rowDatas : []; - this.stores.forEach(s => s.loadData(storeRows)); + stores.forEach(s => s.loadData(storeRows)); } private updateResults() { @@ -582,7 +595,14 @@ export class View buckets: Record = {}, ret: BaseRow[] = []; - dependentFields.forEach(it => this._bucketDependentFields.add(it)); + dependentFields.forEach(it => { + if (this._bucketDependentFields.has(it)) return; + throwIf( + this.getField(it)?.isCalculated, + `BucketSpec 'dependentFields' may not include calculated field '${it}' - calculated values hold no stored slot to diff for re-bucketing.` + ); + this._bucketDependentFields.add(it); + }); // Determine which bucket to put this row into (if any) rows.forEach(row => { @@ -672,8 +692,8 @@ export class View this._records = cube.store._filtered.withFilter(query.filter, this._records); } - private createAggregationContext() { - this._aggContext = new AggregationContext(this); + private createAggregationContext(getFilteredRecords: () => StoreRecord[]) { + this._aggContext = new AggregationContext(this, getFilteredRecords); } /** @@ -695,32 +715,10 @@ export class View return !this.aggregatorsAreSimple || !isEmpty(this._canAggregateFnFieldsByDepth[0]); } - private parseStores(stores: Some): Store[] { - const ret = castArray(stores); - - throwIf( - ret.some(s => s.digestSpec != null && s.digestSpec !== 'cubeRowDigest'), - '`Store.digestSpec` cannot be configured on a Store connected to a Cube View - the View manages record reuse automatically, installing its own row-based digest. Leave unset.' - ); - ret.forEach(s => (s.digestSpec = 'cubeRowDigest')); - - throwIf( - ret.some(s => s.idEncodesTreePath), - '`Store.idEncodesTreePath` cannot be configured on a Store connected to a Cube View - view row ids do not encode a fixed tree position. Leave unset.' - ); - - if (ret.some(s => s.projectionOnly == null && !s.processRawData)) { - this.logWarn( - 'Connected store(s) do not set `projectionOnly` - recommended for improved performance when no additional record parsing or local data modification is required. Set explicitly to false to opt out and silence this warning.' - ); - } - - return ret; - } - override destroy() { instanceManager.unregisterView(this); this.disconnect(); + this.stores = []; super.destroy(); } } diff --git a/data/cube/aggregate/AggregationContext.ts b/data/cube/aggregate/AggregationContext.ts index 149af56e7b..0291773e6e 100755 --- a/data/cube/aggregate/AggregationContext.ts +++ b/data/cube/aggregate/AggregationContext.ts @@ -36,6 +36,10 @@ export class AggregationContext { */ activeField: CubeField = null; + // Filtered-records source supplied by the owning View - read lazily, memoized per context. + private readonly getRecordsFn: () => StoreRecord[]; + private _filteredRecords: StoreRecord[] = null; + /** * Row currently being aggregated, or null if not within a call to an aggregator. * @internal @@ -45,24 +49,25 @@ export class AggregationContext { /** * All records currently meeting the filter for this view. * - * Available only when an aggregator on the view overrides - * {@link Aggregator.dependsOnChildrenOnly} to return false. - * Views with children-only aggregators update incrementally without - * refreshing that collection, so reading it here throws. + * Available to calculated field functions ({@link CubeFieldSpec.calculatedFn}) and to + * aggregators that override {@link Aggregator.dependsOnChildrenOnly} to return false. + * Reading from a children-only aggregator throws - such aggregators must declare their + * wider dependency for the View to keep this collection consistent with their reads. */ get filteredRecords(): StoreRecord[] { - const {activeField, view} = this; + const {activeField} = this; if (activeField?.aggregator.dependsOnChildrenOnly) { throw XH.exception( `The aggregator for the '${activeField.name}' field read \`filteredRecords\`, but does not override \`dependsOnChildrenOnly\` to return false - aggregators depending on records beyond their own children must do so.` ); } - return view._records.list; + return (this._filteredRecords ??= this.getRecordsFn()); } - constructor(view: View) { + constructor(view: View, getRecordsFn: () => StoreRecord[]) { this.view = view; this.appData = {}; + this.getRecordsFn = getRecordsFn; } /** diff --git a/data/cube/impl/RowDataGenerator.ts b/data/cube/impl/RowDataGenerator.ts index 64bfd08454..61200f5c7f 100644 --- a/data/cube/impl/RowDataGenerator.ts +++ b/data/cube/impl/RowDataGenerator.ts @@ -6,44 +6,53 @@ */ import {PlainObject} from '@xh/hoist/core'; -import {isEqual} from 'lodash'; +import { + installCalculatedFieldGetters, + installSourceFieldGetters +} from '@xh/hoist/data/impl/FieldGetterSupport'; +import {shallowEqualArrays} from '@xh/hoist/utils/impl'; +import type {CubeField} from '../CubeField'; import type {View} from '../View'; import {ViewRowData} from '../ViewRowData'; /** * Generates the `ViewRowData` objects published by a View. Owned by its View, with - * query-dependent templates rebuilt when the query's field set or leaf exposure changes. The + * query-dependent classes rebuilt when the query's field set or leaf exposure changes. The * View stamps each minted or mutated row with its monotonic `cubeRowDigest` post-construction - * every row is minted with a `cubeRowDigest` slot so the stamp is an overwrite, never a * shape-changing property add. * * Row shapes are fixed per query, keeping them in V8's compact fast-properties mode rather than * "dictionary mode": - * - Aggregate and bucket row data is cloned from a shared template carrying a slot for every - * ViewRowData property and query field. Rows are only ever written via overwrites of these - * slots - never property adds. + * - Aggregate and bucket row data objects are instances of a generated class whose constructor + * assigns a slot for every ViewRowData property and aggregable query field in a fixed order. + * Rows are only ever written via overwrites of these slots - never property adds. * - Exposed-leaf row data holds no per-leaf copy of field values - queried fields are read * through prototype getters over an own `_src` reference to the leaf's cube record data. One * generated class per query keeps all leaf datas on a single shape with monomorphic, * inlinable reads. * + * Calculated fields (`CubeFieldSpec.calculatedFn`) hold no slot on either class - their values + * are read through prototype getters computing against the View's current AggregationContext, + * so they are always current and never participate in slot writes or digest bumps. + * * @internal */ export class RowDataGenerator { private view: View; - private fieldNames: string[]; + private fields: CubeField[]; private exposesLeaves: boolean; - private parentDataTemplate: ViewRowData = null; - private leafDataClass: LeafDataClass = null; + private parentDataClass: GeneratedDataClass = null; + private leafDataClass: GeneratedLeafDataClass = null; constructor(view: View) { this.view = view; this.init(); } - /** Create a new aggregate or bucket row data object as a clone of the shared template. */ + /** Create a new aggregate or bucket row data object. */ newParentRowData(id: string): ViewRowData { - return {...this.parentDataTemplate, id}; + return new this.parentDataClass(id); } /** Create an exposed-leaf data object - fields read via prototype getters over `src`. */ @@ -57,7 +66,7 @@ export class RowDataGenerator { onQueryChange() { const {view} = this; if ( - !isEqual(view.fieldNames, this.fieldNames) || + !shallowEqualArrays(view.fields, this.fields) || view.exposesLeaves !== this.exposesLeaves ) { this.init(); @@ -65,46 +74,66 @@ export class RowDataGenerator { } private init() { - this.fieldNames = this.view.fieldNames; + this.fields = this.view.fields; this.exposesLeaves = this.view.exposesLeaves; - this.parentDataTemplate = this.buildParentDataTemplate(); + this.parentDataClass = this.buildParentDataClass(); this.leafDataClass = this.buildLeafDataClass(); } - private buildParentDataTemplate(): ViewRowData { - const rowData: PlainObject = { - id: null, - cubeRowType: null, - cubeLabel: null, - cubeDimension: null, - cubeBuckets: null, - children: null, - isCubeLeaf: false, - cubeRowDigest: null, - _cubeLeafChildren: null - }; - this.view.fields.forEach(({name}) => (rowData[name] = null)); - - // Convert into V8 fast-properties mode that we'll need to mint additional fast objects - return {...rowData} as ViewRowData; + private buildParentDataClass(): GeneratedDataClass { + const {view} = this, + {_nonCalcFields, _calcFields} = view, + slotNames = _nonCalcFields.map(it => it.name); + + class ParentRowData extends BaseParentRowData { + constructor(id: string) { + super(id); + // Constructor assignments in fixed order - all instances share one shape. + for (let i = 0; i < slotNames.length; i++) this[slotNames[i]] = null; + } + } + installCalculatedFieldGetters(ParentRowData.prototype, _calcFields, () => view._aggContext); + return ParentRowData; } - private buildLeafDataClass(): LeafDataClass { + private buildLeafDataClass(): GeneratedLeafDataClass { if (!this.exposesLeaves) return null; + const {view} = this, + {_nonCalcFields, _calcFields} = view; class LeafRowData extends BaseLeafRowData {} - this.view.fields.forEach(({name}) => { - Object.defineProperty(LeafRowData.prototype, name, { - get(this: PlainObject) { - return this._src[name]; - }, - enumerable: true - }); - }); + installSourceFieldGetters( + LeafRowData.prototype, + _nonCalcFields.map(it => it.name) + ); + installCalculatedFieldGetters(LeafRowData.prototype, _calcFields, () => view._aggContext); return LeafRowData; } } +/** + * Fixed portion of a View's aggregate/bucket data class - `buildParentDataClass` extends this + * with a slot per aggregable query field. + */ +class BaseParentRowData implements ViewRowData { + id: string; + cubeRowType: 'leaf' | 'aggregate' | 'bucket' = null; + cubeLabel: string = null; + cubeDimension: string = null; + cubeBuckets: PlainObject = null; + children: ViewRowData[] = null; + isCubeLeaf: boolean = false; + cubeRowDigest: number = null; + _cubeLeafChildren: ViewRowData[] = null; + + // Type-only, erased: the interface's index signature. + [key: string]: any; + + constructor(id: string) { + this.id = id; + } +} + /** * Fixed portion of a View's exposed-leaf data class - `buildLeafDataClass` extends this with * per-query field getters reading through the own `_src` reference to the leaf's cube record @@ -141,4 +170,5 @@ class BaseLeafRowData implements ViewRowData { } } -type LeafDataClass = new (id: string, src: PlainObject) => ViewRowData; +type GeneratedDataClass = new (id: string) => ViewRowData; +type GeneratedLeafDataClass = new (id: string, src: PlainObject) => ViewRowData; diff --git a/data/impl/FieldGetterSupport.ts b/data/impl/FieldGetterSupport.ts new file mode 100644 index 0000000000..fdbfa08a37 --- /dev/null +++ b/data/impl/FieldGetterSupport.ts @@ -0,0 +1,63 @@ +/* + * This file belongs to Hoist, an application development toolkit + * developed by Extremely Heavy Industries (www.xh.io | info@xh.io) + * + * Copyright © 2026 Extremely Heavy Industries Inc. + */ + +import {PlainObject} from '@xh/hoist/core'; +import type {Field} from '../Field'; + +/** + * Shared support for building generated data classes whose values are read through prototype + * getters - the exposed-leaf getter-class technique from the Cube's `RowDataGenerator`, + * generalized here for consumption by both the Store and Cube layers. + * + * Getters keep all data objects produced by a Store or View on a single generated shape with + * monomorphic, inlinable reads, and are the delivery mechanism for calculated fields - values + * computed lazily at read time and therefore always current, never stored, and excluded by + * construction from the own-property equality and digest contracts used to detect changed + * records. + * + * @internal + */ + +/** + * Install a prototype getter for each calculated field, computing its value at read time via + * the field's `calculatedFn`. + * + * The getter passes the data object itself as the function's first argument, with the layer + * context (Store or AggregationContext) supplied live by `getContext` - so a value read after + * later context replacement always computes against the current context. + */ +export function installCalculatedFieldGetters( + target: object, + fields: Field[], + getContext: () => any +) { + fields.forEach(({name, calculatedFn}) => { + Object.defineProperty(target, name, { + get(this: PlainObject) { + return calculatedFn(this, getContext()); + }, + enumerable: true, + configurable: true + }); + }); +} + +/** + * Install a prototype getter for each named field, reading through an own `_src` reference to + * an adopted source data object - avoiding a per-object copy of source values. + */ +export function installSourceFieldGetters(target: object, fieldNames: string[]) { + fieldNames.forEach(name => { + Object.defineProperty(target, name, { + get(this: PlainObject) { + return this._src[name]; + }, + enumerable: true, + configurable: true + }); + }); +} diff --git a/data/impl/RecordDataGenerator.ts b/data/impl/RecordDataGenerator.ts new file mode 100644 index 0000000000..35f709ecb4 --- /dev/null +++ b/data/impl/RecordDataGenerator.ts @@ -0,0 +1,133 @@ +/* + * This file belongs to Hoist, an application development toolkit + * developed by Extremely Heavy Industries (www.xh.io | info@xh.io) + * + * Copyright © 2026 Extremely Heavy Industries Inc. + */ +import {PlainObject} from '@xh/hoist/core'; +import {isEmpty} from 'lodash'; +import type {Field} from '../Field'; +import type {Store} from '../Store'; +import type {StoreRecordId} from '../StoreRecord'; +import {installCalculatedFieldGetters, installSourceFieldGetters} from './FieldGetterSupport'; + +/** + * Generates the constructs backing a Store's record `data` objects - the shared defaults object + * and dense template, plus the generated dense and projection classes carrying calculated field + * getters. Owned by its Store and rebuilt whenever data config changes, mirroring the Cube's + * `RowDataGenerator`. + * + * All representations keep records on fixed shapes in V8's compact fast-properties mode. + * + * @internal + */ +export class RecordDataGenerator { + private store: Store; + + /** Store-layer calculated fields - cube-layer fields compute on View rows, never here. */ + calcFields: Field[]; + /** Fields with stored values, compared where record reuse is detected. */ + equalityFields: Field[]; + hasCalcFields: boolean; + + private dataDefaults: PlainObject; + private dataTemplate: PlainObject; + private denseDataClass: new () => PlainObject; + private projectionDataClass: new (src: PlainObject) => PlainObject; + + constructor(store: Store) { + this.store = store; + this.calcFields = store.fields.filter(it => it.isCalculated && !it.isCubeField); + this.equalityFields = store.fields.filter(it => !it.isCalculated); + this.hasCalcFields = !isEmpty(this.calcFields); + + this.dataDefaults = this.createDataDefaults(); + // Clone for fast-props mode - before installing getters, so the spread cannot see them. + this.dataTemplate = {...this.dataDefaults}; + installCalculatedFieldGetters(this.dataDefaults, this.calcFields, () => store); + + this.denseDataClass = this.createDenseDataClass(); + this.projectionDataClass = this.createProjectionDataClass(); + } + + /** New sparse data - own properties for populated values only, defaults (and calculated + * getters) via the shared prototype. */ + sparseData(): PlainObject { + return Object.create(this.dataDefaults); + } + + /** New dense data - a slot for every Field, all records on one fixed shape. Template clones + * sidestep dictionary-mode adds (overwrites are not adds); the generated class keeps that + * guarantee via constructor slack tracking while adding getters a plain clone cannot see. */ + denseData(): PlainObject { + const {denseDataClass} = this; + return denseDataClass ? new denseDataClass() : {...this.dataTemplate}; + } + + /** Data adopting a projection raw - the raw itself, or, with calculated fields, a generated + * getter wrapper over it (`data !== raw`). */ + projectionData(raw: PlainObject): PlainObject { + const {projectionDataClass} = this; + return projectionDataClass ? new projectionDataClass(raw) : raw; + } + + //------------------------ + // Implementation + //------------------------ + // Shared prototype/template source - a defaultValue slot per stored Field. Store-layer + // calculated fields hold no slot; their getters are installed on the returned object. + private createDataDefaults(): PlainObject { + const ret = {}; + this.store.fields.forEach(field => { + if (field.isCalculated && !field.isCubeField) return; + ret[field.name] = field.defaultValue; + }); + return ret; + } + + // Null without calculated fields, directing denseData() to the template clone. `id` slots + // here and below are written post-construction by StoreRecord - an overwrite, never an add. + private createDenseDataClass(): new () => PlainObject { + if (!this.hasCalcFields) return null; + + const {store} = this, + names = this.equalityFields.map(it => it.name), + defaultValues = this.equalityFields.map(it => it.defaultValue); + + class DenseData { + id: StoreRecordId = null; + + constructor() { + for (let i = 0; i < names.length; i++) { + this[names[i]] = defaultValues[i]; + } + } + + // Type-only, erased: field slots assigned by name above. + [key: string]: any; + } + installCalculatedFieldGetters(DenseData.prototype, this.calcFields, () => store); + return DenseData; + } + + // Null when not applicable, directing projectionData() to adopt raw objects directly. + private createProjectionDataClass(): new (src: PlainObject) => PlainObject { + if (!this.store.projectionOnly || !this.hasCalcFields) return null; + + const {store} = this; + class ProjectionData { + id: StoreRecordId = null; + _src: PlainObject; + + constructor(src: PlainObject) { + this._src = src; + } + } + installSourceFieldGetters( + ProjectionData.prototype, + this.equalityFields.map(it => it.name) + ); + installCalculatedFieldGetters(ProjectionData.prototype, this.calcFields, () => store); + return ProjectionData; + } +} diff --git a/docs/README.md b/docs/README.md index 5e9c5e57b9..5cccf2b51c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -170,6 +170,7 @@ breaking changes, before/after code examples, and verification checklists. | Version | Released | Difficulty | Key Changes | |---------|----------|------------|-------------| +| [v89](./upgrade-notes/v89-upgrade-notes.md) | TBD | 🟢 LOW | Connected stores construct with `StoreConfig.view` and are always `projectionOnly`; calculated fields (`FieldSpec.calculatedFn` / `CubeFieldSpec.calculatedFn`) are the new-feature headline | | [v88](./upgrade-notes/v88-upgrade-notes.md) | 2026-09-28 | 🔴 HIGH | TC39 decorators (`accessor`, no `makeObservable`) + dev-utils 16 / Rsbuild, MobX 7 named exports, AG Grid 36 + Theming API, React 19.3, v86 scheduled removals | | [v87](./upgrade-notes/v87-upgrade-notes.md) | TBD | 🟠 MEDIUM | React 19 + Floating UI popovers, data-layer perf overhaul (`leafMap`, `getCubeLeaves`, `StoreRecord.data` access), new column chooser + `RowDragModule`, hoist-core >= 40.5.0 | | [v86](./upgrade-notes/v86-upgrade-notes.md) | 2026-06-12 | 🟠 MEDIUM | AG Grid 34→35, CodeInput → CodeMirror v6 (`mode`→`language`), FileChooser redesign, mobile DateInput native picker, `Runner` API + `withSpan` deprecation | diff --git a/docs/doc-registry.json b/docs/doc-registry.json index c8f0706adc..b7ff771f94 100644 --- a/docs/doc-registry.json +++ b/docs/doc-registry.json @@ -526,6 +526,14 @@ "viewerCategory": "upgrade", "description": "Upgrade guide from v87.x to v88.0.0. High difficulty.", "keywords": ["upgrade", "migration", "breaking changes", "v88", "v87", "experimentalDecorators", "makeObservable", "configureWebpack", "webpack.config.js", "ag-theme-balham", "PopoverFilterChooser", "mergePersistOptions", "LogSource", "localDateCol"] + }, + { + "id": "docs/upgrade-notes/v89-upgrade-notes.md", + "title": "v89 Upgrade Notes", + "mcpCategory": "devops", + "viewerCategory": "upgrade", + "description": "Upgrade guide from v88.x to v89.0.0. Low difficulty.", + "keywords": ["upgrade", "migration", "breaking changes", "v89", "v88", "calculatedFn", "projectionOnly", "connected stores", "StoreConfig.view"] } ] } diff --git a/docs/upgrade-notes/v89-upgrade-notes.md b/docs/upgrade-notes/v89-upgrade-notes.md new file mode 100644 index 0000000000..a1a919b87d --- /dev/null +++ b/docs/upgrade-notes/v89-upgrade-notes.md @@ -0,0 +1,66 @@ +# Hoist React v89 Upgrade Notes + +> **From:** v88.x → v89.0.0 | **Released:** TBD | **Difficulty:** 🟢 LOW + +## Overview + +Hoist React v89 is under active development - this document will grow as breaking changes land. + +The headline addition is **calculated fields** (`FieldSpec.calculatedFn` and +`CubeFieldSpec.calculatedFn`) - client-computed field values at both the Store and Cube View +layers. See the CHANGELOG for details; calculated fields are additive and require no app changes. + +## Connected stores: construct with `StoreConfig.view` + +Stores now connect to a Cube `View` at construction, replacing `ViewConfig.stores` and +`View.setStores()`. A connected store is built as a read-only projection of the View's published +rows - `projectionOnly` and a row-based `digestSpec` are set automatically, view rows are adopted +as record `data` by reference (no re-parse or copy), and conflicting config +(`projectionOnly: false`, `processRawData`, `digestSpec`, `idEncodesTreePath`) throws. + +```typescript +// Before - store built first, then adopted by the View. +const gridModel = new GridModel({store: {fields: [...]}, ...}); +const view = cube.createView({query, stores: gridModel.store, connect: true}); + +// After - View first; the store connects (and loads) at construction. +const view = cube.createView({query, connect: true}); +const gridModel = new GridModel({store: {view, fields: [...]}, ...}); +``` + +- `modifyRecords()` (and other local modification APIs) throw on a projection store - route edits + through `Cube.modifyRecordsAsync()` instead, so they survive view regeneration. +- Replace a connected store's `processRawData` transform with Cube fields or calculated fields + (`FieldSpec.calculatedFn` / `CubeFieldSpec.calculatedFn`) - note `updateData()` never applied + `processRawData`, so such transforms only ever ran on full loads and were stale on updates. +- To detach and later re-attach a connected store (it retains its config and last-loaded data), + use `View.removeStore()` / `addStore()`. To pause updates - e.g. for a hidden tab - prefer + `View.disconnect()` / `connect()`, which stops the whole pipeline. +- `projectionOnly` remains a fully supported opt-in config for ordinary (non-connected) stores. + +## Connected store fields flow from the View + +A connected store's `fields` are now reconciled to its View's query fields, at connection and on +query changes. View-published data is described by the query's own `CubeField` instances - types, +`displayName`s, and calculated status flow through to everything reading field metadata off the +store (`StoreFilterField`, grid filter choosers, column editability) rather than being +independently and typically more weakly declared (e.g. grid-inferred `type: 'auto'` fields). + +Most apps need no changes and simply get stronger metadata. Two cases to review: + +```typescript +// Before - view-published fields redeclared on the connected store for typing/display. +store: {fields: [{name: 'commission', type: 'number', displayName: 'Comm.'}, 'cubeDimension']} + +// After - view-published fields are adopted automatically; declare only store-layer extras. +// Customize display metadata for view-published fields on the CubeField itself. +store: {fields: ['cubeDimension']} +``` + +- An app field sharing a view field's name is superseded by the view's `CubeField` - move any + custom `displayName` or other metadata for such fields onto the Cube's field definition. +- A store-layer `calculatedFn` field sharing a view field's name now throws at connection - both + would claim to compute the value. Rename the store-layer field, or compute it on the View via + `CubeFieldSpec.calculatedFn`. +- `Store.fields` visibly changes at connection and on `View.updateQuery` - code capturing a + connected store's field list at construction should read it lazily instead.