diff --git a/Cargo.lock b/Cargo.lock index c6dde4b784e..605a981e8ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2975,7 +2975,7 @@ dependencies = [ [[package]] name = "grovedb" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=97250247423ddc7fcc2865ea0025fc5bd41d0883#97250247423ddc7fcc2865ea0025fc5bd41d0883" +source = "git+https://github.com/dashpay/grovedb?rev=d548e28228e2aca1361d9b12835388be0f811a92#d548e28228e2aca1361d9b12835388be0f811a92" dependencies = [ "axum 0.8.9", "bincode", @@ -3014,7 +3014,7 @@ dependencies = [ [[package]] name = "grovedb-bulk-append-tree" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=97250247423ddc7fcc2865ea0025fc5bd41d0883#97250247423ddc7fcc2865ea0025fc5bd41d0883" +source = "git+https://github.com/dashpay/grovedb?rev=d548e28228e2aca1361d9b12835388be0f811a92#d548e28228e2aca1361d9b12835388be0f811a92" dependencies = [ "bincode", "blake3", @@ -3032,7 +3032,7 @@ dependencies = [ [[package]] name = "grovedb-commitment-tree" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=97250247423ddc7fcc2865ea0025fc5bd41d0883#97250247423ddc7fcc2865ea0025fc5bd41d0883" +source = "git+https://github.com/dashpay/grovedb?rev=d548e28228e2aca1361d9b12835388be0f811a92#d548e28228e2aca1361d9b12835388be0f811a92" dependencies = [ "blake3", "grovedb-bulk-append-tree", @@ -3049,7 +3049,7 @@ dependencies = [ [[package]] name = "grovedb-costs" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=97250247423ddc7fcc2865ea0025fc5bd41d0883#97250247423ddc7fcc2865ea0025fc5bd41d0883" +source = "git+https://github.com/dashpay/grovedb?rev=d548e28228e2aca1361d9b12835388be0f811a92#d548e28228e2aca1361d9b12835388be0f811a92" dependencies = [ "integer-encoding", "intmap", @@ -3059,7 +3059,7 @@ dependencies = [ [[package]] name = "grovedb-dense-fixed-sized-merkle-tree" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=97250247423ddc7fcc2865ea0025fc5bd41d0883#97250247423ddc7fcc2865ea0025fc5bd41d0883" +source = "git+https://github.com/dashpay/grovedb?rev=d548e28228e2aca1361d9b12835388be0f811a92#d548e28228e2aca1361d9b12835388be0f811a92" dependencies = [ "bincode", "blake3", @@ -3073,7 +3073,7 @@ dependencies = [ [[package]] name = "grovedb-element" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=97250247423ddc7fcc2865ea0025fc5bd41d0883#97250247423ddc7fcc2865ea0025fc5bd41d0883" +source = "git+https://github.com/dashpay/grovedb?rev=d548e28228e2aca1361d9b12835388be0f811a92#d548e28228e2aca1361d9b12835388be0f811a92" dependencies = [ "bincode", "bincode_derive", @@ -3089,7 +3089,7 @@ dependencies = [ [[package]] name = "grovedb-epoch-based-storage-flags" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=97250247423ddc7fcc2865ea0025fc5bd41d0883#97250247423ddc7fcc2865ea0025fc5bd41d0883" +source = "git+https://github.com/dashpay/grovedb?rev=d548e28228e2aca1361d9b12835388be0f811a92#d548e28228e2aca1361d9b12835388be0f811a92" dependencies = [ "grovedb-costs", "hex", @@ -3101,7 +3101,7 @@ dependencies = [ [[package]] name = "grovedb-merk" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=97250247423ddc7fcc2865ea0025fc5bd41d0883#97250247423ddc7fcc2865ea0025fc5bd41d0883" +source = "git+https://github.com/dashpay/grovedb?rev=d548e28228e2aca1361d9b12835388be0f811a92#d548e28228e2aca1361d9b12835388be0f811a92" dependencies = [ "bincode", "bincode_derive", @@ -3127,7 +3127,7 @@ dependencies = [ [[package]] name = "grovedb-merkle-mountain-range" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=97250247423ddc7fcc2865ea0025fc5bd41d0883#97250247423ddc7fcc2865ea0025fc5bd41d0883" +source = "git+https://github.com/dashpay/grovedb?rev=d548e28228e2aca1361d9b12835388be0f811a92#d548e28228e2aca1361d9b12835388be0f811a92" dependencies = [ "bincode", "blake3", @@ -3140,7 +3140,7 @@ dependencies = [ [[package]] name = "grovedb-path" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=97250247423ddc7fcc2865ea0025fc5bd41d0883#97250247423ddc7fcc2865ea0025fc5bd41d0883" +source = "git+https://github.com/dashpay/grovedb?rev=d548e28228e2aca1361d9b12835388be0f811a92#d548e28228e2aca1361d9b12835388be0f811a92" dependencies = [ "hex", ] @@ -3148,7 +3148,7 @@ dependencies = [ [[package]] name = "grovedb-private-document-store" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=97250247423ddc7fcc2865ea0025fc5bd41d0883#97250247423ddc7fcc2865ea0025fc5bd41d0883" +source = "git+https://github.com/dashpay/grovedb?rev=d548e28228e2aca1361d9b12835388be0f811a92#d548e28228e2aca1361d9b12835388be0f811a92" dependencies = [ "blake3", "grovedb-bulk-append-tree", @@ -3161,7 +3161,7 @@ dependencies = [ [[package]] name = "grovedb-query" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=97250247423ddc7fcc2865ea0025fc5bd41d0883#97250247423ddc7fcc2865ea0025fc5bd41d0883" +source = "git+https://github.com/dashpay/grovedb?rev=d548e28228e2aca1361d9b12835388be0f811a92#d548e28228e2aca1361d9b12835388be0f811a92" dependencies = [ "bincode", "byteorder", @@ -3177,7 +3177,7 @@ dependencies = [ [[package]] name = "grovedb-storage" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=97250247423ddc7fcc2865ea0025fc5bd41d0883#97250247423ddc7fcc2865ea0025fc5bd41d0883" +source = "git+https://github.com/dashpay/grovedb?rev=d548e28228e2aca1361d9b12835388be0f811a92#d548e28228e2aca1361d9b12835388be0f811a92" dependencies = [ "blake3", "grovedb-costs", @@ -3196,7 +3196,7 @@ dependencies = [ [[package]] name = "grovedb-version" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=97250247423ddc7fcc2865ea0025fc5bd41d0883#97250247423ddc7fcc2865ea0025fc5bd41d0883" +source = "git+https://github.com/dashpay/grovedb?rev=d548e28228e2aca1361d9b12835388be0f811a92#d548e28228e2aca1361d9b12835388be0f811a92" dependencies = [ "thiserror 2.0.18", "versioned-feature-core 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3205,7 +3205,7 @@ dependencies = [ [[package]] name = "grovedb-visualize" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=97250247423ddc7fcc2865ea0025fc5bd41d0883#97250247423ddc7fcc2865ea0025fc5bd41d0883" +source = "git+https://github.com/dashpay/grovedb?rev=d548e28228e2aca1361d9b12835388be0f811a92#d548e28228e2aca1361d9b12835388be0f811a92" dependencies = [ "hex", "itertools 0.14.0", @@ -3214,7 +3214,7 @@ dependencies = [ [[package]] name = "grovedbg-types" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=97250247423ddc7fcc2865ea0025fc5bd41d0883#97250247423ddc7fcc2865ea0025fc5bd41d0883" +source = "git+https://github.com/dashpay/grovedb?rev=d548e28228e2aca1361d9b12835388be0f811a92#d548e28228e2aca1361d9b12835388be0f811a92" dependencies = [ "serde", "serde_with 3.21.0", diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index 8589d4516af..3f0a1e4cac5 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -68,6 +68,7 @@ - [Average Index Examples](drive/average-index-examples.md) - [Document Ranked Trees](drive/document-ranked-trees.md) - [Ranked Index Examples](drive/ranked-index-examples.md) +- [Time-Range Index TTL](drive/time-range-ttl.md) - [Index-Only Document Types](drive/index-only-document-types.md) # Testing diff --git a/book/src/drive/time-range-ttl.md b/book/src/drive/time-range-ttl.md new file mode 100644 index 00000000000..37958440630 --- /dev/null +++ b/book/src/drive/time-range-ttl.md @@ -0,0 +1,224 @@ +# Time-Range Index TTL + +Architecture reference for the `ttl` key of `timeRange` indexes: what it +means, how expired windows are drained, and the invariants every walker +shares. The storage primitive underneath is grovedb's flat-subtree drop +([dashpay/grovedb#848](https://github.com/dashpay/grovedb/issues/848), +landed in grovedb PR #849); see +[the storage section](#grovedb-dependency-flat-subtree-drop). + +## Motivation + +A `timeRange` index stores every document once per containing window, and +a ranked one additionally rewrites a per-window secondary on every write. +All of those bytes are billed as **storage** — a price that prepays +~perpetual retention through the epoch-distribution model — even though +windowed data is intrinsically ephemeral: a "posts liked this hour" +bucket is worthless once the trending surface has moved past it. The +result is that the flagship use case (likes feeding a trending index) +pays perpetuity prices for state with a useful life measured in days, +multiplied by the grid's overlap factor. + +Nobody cleans this up, either. Deletion costs the deleter processing, +refunds accrue to owners who have no reason to come back for entries this +small, and the state lingers forever. + +## Semantics + +A `timeRange` index may declare a **time to live**: + +```json +"timeRange": { "on": "$createdAt", "range": 3600, "step": 3600, "ttl": 604800 } +``` + +In one paragraph: entries under this index exist for at most `ttl` +seconds past their bucket's start, plus a bounded drainage lag. Expired +buckets are drained **lazily, on write**: every state transition that +writes into the index continues draining the oldest expired bucket, +deepest-first, under a per-write operation budget. A fully drained +window is provably absent, exactly like a window that never held +documents. An expired window is **not queryable at all** — `byStart` +rejects starts past the horizon, so the drainage lag is purely internal: +drainage only ever touches expired buckets, which makes every window a +query can address complete. Everything written under the index's +grid-qualified level +bills as **processing, not storage** — including the transitional bytes +— at an ephemeral-bytes rate. + +### Why the fee reclassification is honest, not a subsidy + +Storage fees prepay retention distributed across future epochs — decades +of it. A byte that provably lives at most one week consumes on the order +of **1/2,600th** of that retention. The real resource cost of a TTL'd +write is compute and write amplification (already processing) plus a +week of disk occupancy, which a flat per-byte processing surcharge covers +safely *because `ttl` is capped*. Version 1 caps it at **one week** +(`SystemLimits::max_time_range_ttl_seconds = 604 800`). + +The load-bearing simplification: **TTL'd subtrees never create +refundable storage.** No `StorageFlags`, no owner/epoch refund entries. +That single property pays off three times: + +1. the fee reroute needs no refund-ledger reconciliation; +2. cleanup owes nobody anything; +3. deletion needs no byte metering for consensus — which is what makes + O(1) bucket drops possible at all (see the grovedb dependency). + +## Grammar and validation + +- `ttl` is an optional key of the `timeRange` map, in seconds, parsed + into the transform. It is **not part of the grid identity**: + [`TimeRangeTransform::storage_key`] excludes it, so declaring or + changing a TTL never forks the storage level, and query-side grid + matching ([`TimeRangeGridSpec`]) continues to compare + `(range, step, phase)` only. +- **`ttl ≥ range`.** `$createdAt` is consensus-assigned from block time, + so writes only ever target windows containing *now*; this invariant + guarantees no bucket that can still receive entries (or serve as the + `oldest` selector's window) is ever dropped. +- **`ttl ≤ SystemLimits::max_time_range_ttl_seconds`** (one week in v1). + The cap is what makes the flat ephemeral-byte rate safe. +- **One TTL per grid per field.** Two indexes bucketing the same field + with the same grid share one storage level; a differing `ttl` would + give the shared subtree two conflicting lifecycles. Rejected at + contract validation. +- Composes with everything the grid already composes with: `countable`, + the range axes, ranked levels below the bucket, `unique` + (`range == step`, `$createdAt`), indexOnly document types. + `preallocated` stays banned with `timeRange` for the pre-existing + structural reason. + +## Cleanup + +**Trigger** — deterministic and write-amortized: **every write** into a +TTL'd index continues drainage of the oldest expired bucket (start +`< block_time − ttl`), deepest-first, spending at most +`SystemLimits::max_time_range_ttl_drop_operations_per_write` O(1) drop +operations and resuming exactly where the previous write's budget ran +out. When nothing is expired, the check is a single bounded range read. +The operation count of a full bucket scales with its distinct groups, +and write volume scales with group volume, so drainage keeps pace +roughly one window behind; after a quiet spell the backlog amortizes +across subsequent writes instead of dumping a week of demolition on the +first like after a lull. + +**Residue** — an index that never receives another write keeps its final +`ttl` of buckets indefinitely. This is bounded garbage that owes nobody +a refund. If it ever matters, the backstop is an epoch-transition sweep +riding the existing scheduled-cleanup pattern +(`check_for_ended_vote_polls` / `clean_up_after_vote_polls_end`); +deliberately **out of scope for v1**. + +**User deletes and updates of expired documents** — handled at +**full-path granularity**, because a bucket drains piecewise: an entry +whose bucket (or whose group's trees inside a standing bucket) the drain +already took is skipped as cleanly removed; one whose trees still stand +is removed normally, so a not-yet-drained expired bucket never carries +dangling references. Every check is deterministic — it reads consensus +state plus the carried `$createdAt` and block time. Writes never target +expired windows, so an update of a fully expired document simply leaves +it without entries under the TTL'd index. + +**Per-index semantics** — TTL removes entries from *this index only*. +An indexOnly like whose windowed entries expire keeps counting in the +all-time ranked `byPost` and in `byLiker`; permanence lives where the +contract declares it. Ranked per-window secondaries die with their +bucket — which also caps live leaderboard state at ~`ttl / step` windows +per index. + +## grovedb dependency: flat-subtree drop + +Dropping a bucket must never put user-scaled work on the consensus path. +The primitive that landed (grovedb PR #849) is the **flat-subtree drop**: +O(1) consensus removal of a subtree *declared to contain no child +subtrees* — an ordinary parent-Merk element delete whose cost is +independent of the subtree's contents — staging a durable redo record +(atomically, outside the root hash) that names every storage prefix the +drop orphaned: the subtree's own and, for indexed primaries, its three +per-axis secondary prefixes. Reclamation is DB-level range tombstones, +drained by `GroveDb::flush_pending_prefix_drops` — idempotent, +crash-safe, snapshot-correct, and never part of consensus cost. + +A time-range bucket is *not* flat, so the platform drains it +**deepest-first, one flat unit at a time** (`drain_expired_time_range_buckets`): + +1. each group's `[0]` reference tree — flat by construction, and where + the mass lives — is flat-dropped; +2. the emptied group value tree leaves through the flat drop — or, under + a ranked (indexed-primary) property-name tree, through grovedb's + dedicated indexed-tree delete, which mirrors the group out of the + ranking secondary; +3. the drained property-name tree is flat-dropped (dooming its secondary + prefixes when ranked); +4. the emptied bucket is flat-dropped. + +Every step is O(1); the *number* of steps scales with the window's +distinct groups, and that count is what +`SystemLimits::max_time_range_ttl_drop_operations_per_write` bounds. +**Every write** into a TTL'd index continues drainage where the previous +budget stopped (when nothing is expired, the check is one bounded range +read); write volume scales with group volume, so drainage keeps pace +roughly one window behind. Between writes a bucket may stand partially +drained — within TTL semantics (entries live *at most* `ttl`) — and the +removal walkers handle those states at full-path granularity: a +document whose group the drain already took deletes as a clean skip, +one whose group still stands is removed normally. + +The flat-drop path-reuse contract (never re-create a dropped path before +its record drains) holds by construction: bucket paths embed their +window start, and writes never target expired windows. The host side: +drive-abci calls `flush_pending_prefix_drops` after committing each +block's transaction and once at startup, completing reclamation a crash +may have interrupted. + +## Fee mechanics + +Write operations targeting a TTL'd index's subtrees are classified +**ephemeral**: the walkers route them into a separate operation batch +(`LowLevelDriveOperation::EphemeralGroveOperation`), applied after the +standing batch, whose captured cost is consumed on its own terms — added +bytes bill to **processing** at +`FeeStorageVersion::ttl_ephemeral_disk_usage_credit_per_byte` +(270 credits/byte, 1% of the storage rate, ~27× a pro-rata week of +retention) and the storage fee contribution is **zero**. The elements +carry no storage flags, so deletion — the TTL drain or a user delete of +a not-yet-expired document — is basic removal with no refund entries: +there is nothing to refund, which is also where TTL writers collectively +pre-pay the drainage described below. Cost estimation routes through the +same split, so estimated and actual fees stay in the same class. + +Drainage itself and the walkers' TTL bookkeeping reads are **unbilled**: +their costs go to scratch accounting, never to the triggering user. That +is load-bearing for the `estimated >= actual` fee invariant — the +estimation dry run cannot read state and therefore cannot price +state-dependent drainage, so billing it only on execution would let a +transition pass validation and then overdraw on apply. The unbilled work +is bounded: a capped count of O(1) drop operations plus a handful of +bounded reads per write. + +## Queries + +Unchanged in shape, with one hard rule: on a TTL'd index, **expired +windows are not queryable**. `byStart` resolution rejects any start past +the expiry horizon (the same strictly-below predicate the drain uses), +on the server from committed block time and on the verifier from the +quorum-signed response `time_ms` — so a node cannot serve an expired +window's remnants past a verifying client. The point of the gate is that +a mid-drainage window would otherwise serve a truncated answer that +looks authoritative; rejecting the question is deterministic where +"whatever the drain has left" is not. Because drainage only ever touches +expired buckets, every window the resolver admits is **complete**, and a +window a past drain fully emptied inside its lifetime never existed — +absence proves normally. The relative selectors (`newest` / `oldest`) +can never address an expired window at all (`ttl >= range` guarantees +it). + +## Versioning + +Everything rides the still-unreleased PV14 grammar: the `ttl` key joins +the meta-schema v3 `timeRange` map, the two limits join the (also +unreleased) `SYSTEM_LIMITS_V4` in place, and the ephemeral-bytes rate +joins the shared storage fee table directly — no fee-version fork, +because the rate is dead below PV14 (the grammar does not parse, so no +ephemeral-classified operation can exist to read it). No migration +story exists or is needed. diff --git a/packages/rs-dpp/Cargo.toml b/packages/rs-dpp/Cargo.toml index b844798fdf1..74ae2f26713 100644 --- a/packages/rs-dpp/Cargo.toml +++ b/packages/rs-dpp/Cargo.toml @@ -71,7 +71,7 @@ strum = { version = "0.26", features = ["derive"] } json-schema-compatibility-validator = { path = '../rs-json-schema-compatibility-validator', optional = true } once_cell = "1.19.0" tracing = { version = "0.1.41" } -grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "97250247423ddc7fcc2865ea0025fc5bd41d0883", optional = true } +grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "d548e28228e2aca1361d9b12835388be0f811a92", optional = true } [dev-dependencies] tokio = { version = "1.40", features = ["full"] } diff --git a/packages/rs-dpp/schema/meta_schemas/document/v3/document-meta.json b/packages/rs-dpp/schema/meta_schemas/document/v3/document-meta.json index 34649a06f76..51e6b34c648 100644 --- a/packages/rs-dpp/schema/meta_schemas/document/v3/document-meta.json +++ b/packages/rs-dpp/schema/meta_schemas/document/v3/document-meta.json @@ -666,6 +666,11 @@ "type": "integer", "minimum": 0, "description": "Grid alignment phase, in seconds. Range starts are `phase + k * step`; must be strictly less than `step` (a larger value would be a redundant spelling of `phase % step`) and strictly less than one year (31536000 — a phase further out could sit past current block time on a huge step, leaving valid timestamps before the grid's first bucket). A pure alignment offset — it moves where window boundaries fall (e.g. daily windows cut at 06:00 UTC instead of midnight) and never excludes any real timestamp. Defaults to 0." + }, + "ttl": { + "type": "integer", + "minimum": 1, + "description": "Time to live, in seconds: entries under this index exist for at most this long past their bucket's start, plus a bounded drainage lag — every write into the index continues draining the oldest expired bucket under a per-write operation budget, and expired windows are not queryable (a byStart selection past the horizon is rejected), so every queryable window is complete. Must be at least `range` (a window still able to receive consensus-timestamped writes can never expire) and at most a protocol-versioned cap (604800 — one week — at protocol version 14). Indexes bucketing one field with the same grid share its storage level and must declare the same ttl. Bytes written under a TTL'd index bill to processing at an ephemeral-bytes rate instead of to storage, carry no storage flags, and refund nothing on removal. Omitted means entries live forever. Available from protocol version 14." } }, "required": ["on", "range", "step"], diff --git a/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/common/mod.rs b/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/common/mod.rs index 5b21ea1d724..748ce4a3ff2 100644 --- a/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/common/mod.rs +++ b/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/common/mod.rs @@ -928,6 +928,40 @@ fn parse_indices( )); } } + // The TTL cap is likewise a versioned system + // limit — it is what makes billing TTL'd bytes + // at a flat processing rate honest, so retuning + // it is a protocol-version decision. The lower + // bound (`ttl >= range`) is structural and + // checked in `Index` parsing. + if let Some(ttl_seconds) = transform.ttl_seconds { + if let Some(max_ttl) = ctx + .platform_version + .system_limits + .max_time_range_ttl_seconds + { + if ttl_seconds > max_ttl { + return Err(consensus_or_protocol_data_contract_error( + DataContractError::InvalidContractStructure(format!( + "timeRange.ttl ({} seconds) exceeds the maximum \ + of {} seconds: the flat ephemeral-storage \ + pricing TTL'd entries bill under is only an \ + honest rate while the lifetime it covers is \ + bounded", + ttl_seconds, max_ttl + )), + )); + } + } else { + return Err(consensus_or_protocol_data_contract_error( + DataContractError::InvalidContractStructure( + "timeRange.ttl is not supported by this protocol \ + version" + .to_string(), + ), + )); + } + } let source = transform.source.as_str(); let is_system_timestamp = matches!( source, @@ -1131,9 +1165,45 @@ fn parse_indices( // different grids (or not at all) — each grid forks into its own index // level, keyed by the property name qualified with the grid parameters // (`TimeRangeTransform::storage_key`), so a bucketed level never shares - // a keyspace with a plain level or with another grid's level. No - // cross-index agreement rule is needed; identical grids simply share - // one level. + // a keyspace with a plain level or with another grid's level. The ONE + // cross-index agreement rule is the TTL: it is deliberately excluded + // from the grid identity (declaring or changing it must not fork the + // storage level), so two indexes sharing a grid on one field share one + // level's subtrees — and a level cannot have two lifecycles. Identical + // grids must declare identical TTLs (including both declaring none). + for (name_a, index_a) in indices.iter() { + let Some(transform_a) = &index_a.time_range else { + continue; + }; + for (name_b, index_b) in indices.iter() { + if name_b <= name_a { + continue; + } + let Some(transform_b) = &index_b.time_range else { + continue; + }; + if transform_a.source == transform_b.source + && transform_a.range_seconds == transform_b.range_seconds + && transform_a.step_seconds == transform_b.step_seconds + && transform_a.phase_seconds == transform_b.phase_seconds + && transform_a.ttl_seconds != transform_b.ttl_seconds + { + return Err(consensus_or_protocol_data_contract_error( + DataContractError::InvalidContractStructure(format!( + "indexes \"{}\" and \"{}\" bucket \"{}\" with the same grid but \ + different TTLs ({:?} vs {:?} seconds): indexes sharing a grid share \ + its storage level, and one level cannot have two lifecycles — \ + declare the same ttl on both (or on neither)", + name_a, + name_b, + transform_a.source, + transform_a.ttl_seconds, + transform_b.ttl_seconds + )), + )); + } + } + } let index_structure = IndexLevel::try_from_indices(indices.values(), ctx.name, ctx.platform_version)?; diff --git a/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v3/ranked_prefix_overlap.rs b/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v3/ranked_prefix_overlap.rs index cdd898252f1..e27fd0c6c7c 100644 --- a/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v3/ranked_prefix_overlap.rs +++ b/packages/rs-dpp/src/data_contract/document_type/class_methods/try_from_schema/v3/ranked_prefix_overlap.rs @@ -309,6 +309,7 @@ mod tests { range_seconds: 3_600, step_seconds: 3_600, phase_seconds: 0, + ttl_seconds: None, } } diff --git a/packages/rs-dpp/src/data_contract/document_type/index/mod.rs b/packages/rs-dpp/src/data_contract/document_type/index/mod.rs index b94de9bc5e9..ef11e99925a 100644 --- a/packages/rs-dpp/src/data_contract/document_type/index/mod.rs +++ b/packages/rs-dpp/src/data_contract/document_type/index/mod.rs @@ -1631,6 +1631,7 @@ impl Index { let mut range_seconds: Option = None; let mut step_seconds: Option = None; let mut phase_seconds: u64 = 0; + let mut ttl_seconds: Option = None; for (tr_key_value, tr_value) in time_range_map { let tr_key = tr_key_value @@ -1668,6 +1669,13 @@ impl Index { ) })?; } + "ttl" => { + ttl_seconds = Some(tr_value.to_integer().map_err(|_| { + DataContractError::ValueWrongType( + "timeRange.ttl should be an integer".to_string(), + ) + })?); + } other => { return Err(DataContractError::InvalidContractStructure(format!( "unexpected timeRange field: {}", @@ -1698,6 +1706,7 @@ impl Index { range_seconds, step_seconds, phase_seconds, + ttl_seconds, }); } // `terminal` is guarded the same way as the ranking keywords @@ -2170,6 +2179,25 @@ impl Index { transform.source ))); } + // A TTL below the window length would expire a bucket that can + // still receive writes: `$createdAt` is consensus-assigned from + // block time, so a document entering the grid lands in every + // window containing *now* — the oldest of which started up to + // `range` ago. `ttl >= range` is the invariant that lets the + // cleanup, the delete walker's expired-entry skip and the + // `oldest` selector all assume a live window can never be + // expired. The upper bound (SystemLimits) is checked at the + // document-type level, where the platform version is in scope. + if let Some(ttl_seconds) = transform.ttl_seconds { + if ttl_seconds < transform.range_seconds { + return Err(DataContractError::InvalidContractStructure(format!( + "timeRange.ttl ({} seconds) must be at least the window length \ + ({} seconds): a window still able to receive consensus-timestamped \ + writes can never be expired", + ttl_seconds, transform.range_seconds + ))); + } + } // Same reasoning as the ranked `nullSearchable` rejection above, // plus a write-path invariant: the insert, delete and update // walkers all agree that a null timestamp keeps a single ordinary @@ -2195,6 +2223,7 @@ impl Index { ("range", transform.range_seconds), ("step", transform.step_seconds), ("phase", transform.phase_seconds), + ("ttl", transform.ttl_seconds.unwrap_or(0)), ] { if seconds > u64::MAX / 1_000 { return Err(DataContractError::InvalidContractStructure(format!( @@ -3312,6 +3341,7 @@ mod tests { range_seconds: 86_400, step_seconds: 86_400, phase_seconds: 0, + ttl_seconds: None, }); index } @@ -5024,6 +5054,75 @@ mod tests { ); } + /// `ttl` parses into the transform; a ttl below the window length is + /// structurally rejected (a window still able to receive + /// consensus-timestamped writes can never expire). + #[test] + fn test_index_try_from_time_range_ttl_parses_and_bounds_below() { + let mut index_map = prefix_ranked_index_map(Value::Bool(true)); + index_map[0].1 = Value::Array(vec![ + Value::Map(vec![( + Value::Text("$createdAt".to_string()), + Value::Text("asc".to_string()), + )]), + Value::Map(vec![( + Value::Text("postId".to_string()), + Value::Text("asc".to_string()), + )]), + ]); + index_map.push(( + Value::Text("timeRange".to_string()), + Value::Map(vec![ + ( + Value::Text("on".to_string()), + Value::Text("$createdAt".to_string()), + ), + (Value::Text("range".to_string()), Value::U64(21_600)), + (Value::Text("step".to_string()), Value::U64(21_600)), + (Value::Text("ttl".to_string()), Value::U64(86_400)), + ]), + )); + let index = Index::try_from_value_map(index_map.as_slice(), v3_admissions()) + .expect("a ttl of at least the range parses"); + assert_eq!( + index.time_range.expect("transform present").ttl_seconds, + Some(86_400) + ); + + let mut index_map = prefix_ranked_index_map(Value::Bool(true)); + index_map[0].1 = Value::Array(vec![ + Value::Map(vec![( + Value::Text("$createdAt".to_string()), + Value::Text("asc".to_string()), + )]), + Value::Map(vec![( + Value::Text("postId".to_string()), + Value::Text("asc".to_string()), + )]), + ]); + index_map.push(( + Value::Text("timeRange".to_string()), + Value::Map(vec![ + ( + Value::Text("on".to_string()), + Value::Text("$createdAt".to_string()), + ), + (Value::Text("range".to_string()), Value::U64(21_600)), + (Value::Text("step".to_string()), Value::U64(21_600)), + (Value::Text("ttl".to_string()), Value::U64(21_599)), + ]), + )); + let msg = format!( + "{:?}", + Index::try_from_value_map(index_map.as_slice(), v3_admissions()) + .expect_err("a ttl below the window length must be rejected") + ); + assert!( + msg.contains("must be at least the window length"), + "expected the lower-bound rejection, got {msg}" + ); + } + fn time_range_entry() -> Value { Value::Map(vec![ ( diff --git a/packages/rs-dpp/src/data_contract/document_type/index/time_range.rs b/packages/rs-dpp/src/data_contract/document_type/index/time_range.rs index a2601de04ac..8ac03d1c97d 100644 --- a/packages/rs-dpp/src/data_contract/document_type/index/time_range.rs +++ b/packages/rs-dpp/src/data_contract/document_type/index/time_range.rs @@ -37,14 +37,13 @@ use serde::{Deserialize, Serialize}; /// instant, there is always an active range covering a near-full `range` /// window of history (see [`Self::oldest_active_start`]). /// -/// Note that the *server-ordered* form (`ORDER BY COUNT(*)` — the ranked -/// query surface) cannot yet be combined with a time-range selection: -/// contract validation rejects the ranked keywords on a `timeRange` index -/// (with overlapping windows a document would be ranked into -/// `overlap_factor` groups at once) and the ranked dispatcher rejects -/// time-range selections, so "top K by count within the bucket" is served -/// as the grouped count above with client-side ordering until bucket-aware -/// ranked semantics are deliberately designed. +/// The *server-ordered* form (`ORDER BY COUNT(*)` — the ranked query +/// surface) composes with bucketing since the ranked-windowed work: an +/// index may declare ranked levels **below** the bucketed one, each +/// window's leaderboard is a per-prefix secondary under that window's +/// bucket value tree, and a ranked query pins the window through a +/// resolved time-range selection. Only ranking the bucketed level itself +/// (ordering the windows by their aggregates) remains deferred. /// /// At the GroveDB storage layer, a transformed first property gets its own /// index level keyed by [`Self::storage_key`] — the property name qualified @@ -89,6 +88,25 @@ pub struct TimeRangeTransform { /// daily windows cut at 06:00 UTC instead of midnight). Defaults to `0`. #[cfg_attr(feature = "serde-conversion", serde(rename = "phase", default))] pub phase_seconds: u64, + /// Time to live, in seconds: entries under this index exist for at most + /// this long past their bucket's start, after which the whole bucket is + /// dropped (lazily, by the write that creates a new bucket). `None` + /// means entries live forever, exactly as before the key existed. + /// + /// Deliberately **not part of the grid identity**: [`Self::storage_key`] + /// excludes it, so a TTL never forks the storage level, and query-side + /// grid matching (`TimeRangeGridSpec`) compares `(range, step, phase)` + /// only. Contract validation requires `ttl >= range` (a window still + /// able to receive consensus-timestamped writes, or to serve as the + /// `oldest` selector's window, can never expire) and caps it at + /// `SystemLimits::max_time_range_ttl_seconds` — the cap is what makes + /// billing TTL'd bytes at a flat processing rate honest. Two indexes + /// sharing a grid on one field share its storage level, so they must + /// also share this value (validated). + /// + /// See `book/src/drive/time-range-ttl.md`. + #[cfg_attr(feature = "serde-conversion", serde(rename = "ttl", default))] + pub ttl_seconds: Option, } impl TimeRangeTransform { @@ -224,6 +242,37 @@ impl TimeRangeTransform { .collect() } + /// The time to live on the millisecond timeline, when declared. See + /// [`Self::range_ms`] for why the accessor exists and why it saturates. + pub fn ttl_ms(&self) -> Option { + self.ttl_seconds.map(|ttl| ttl.saturating_mul(1_000)) + } + + /// The TTL horizon at `block_time_ms`: buckets whose start is strictly + /// below the returned value are expired. `None` when the transform + /// declares no TTL (nothing ever expires) — and in the unreachable + /// pre-horizon sliver where `block_time_ms < ttl` (no bucket can be + /// expired before one TTL has elapsed since the epoch). + /// + /// This is the single definition of "expired": the cleanup performed by + /// the insert walker, the expired-entry skip in the delete and update + /// walkers, and any future backstop sweep must all derive the horizon + /// through this function, or one walker deletes entries another still + /// expects to find. + pub fn expiry_horizon_ms(&self, block_time_ms: u64) -> Option { + self.ttl_ms().and_then(|ttl_ms| { + let horizon = block_time_ms.checked_sub(ttl_ms)?; + (horizon > 0).then_some(horizon) + }) + } + + /// Whether a bucket starting at `start_ms` is expired at + /// `block_time_ms`. `false` whenever no TTL is declared. + pub fn bucket_expired(&self, start_ms: u64, block_time_ms: u64) -> bool { + self.expiry_horizon_ms(block_time_ms) + .is_some_and(|horizon| start_ms < horizon) + } + /// Whether the millisecond timestamp `start_ms` is one of this grid's /// range starts — i.e. `phase + k * step` for some `k = 0, 1, 2, …` on /// the millisecond timeline. @@ -331,6 +380,7 @@ mod tests { range_seconds: 6 * HOUR_SECONDS, step_seconds: 2 * HOUR_SECONDS, phase_seconds: 0, + ttl_seconds: None, } } @@ -349,6 +399,7 @@ mod tests { range_seconds: u64::MAX, step_seconds: u64::MAX, phase_seconds: 0, + ttl_seconds: None, }; assert_eq!(t.range_ms(), u64::MAX); assert_eq!(t.overlap_factor(), 1); @@ -378,6 +429,7 @@ mod tests { range_seconds: 60, step_seconds: 20, phase_seconds: 5, + ttl_seconds: None, }; assert_eq!(t.most_recent_start(4_999), None); assert_eq!(t.containing_buckets(4_999), Vec::::new()); @@ -458,6 +510,7 @@ mod tests { range_seconds: 60, step_seconds: 20, phase_seconds: 5, + ttl_seconds: None, }; let raw = DocumentPropertyType::encode_date_timestamp(4_999); assert_eq!(t_phased.entry_keys_for_raw(&raw), Vec::>::new()); @@ -479,6 +532,7 @@ mod tests { range_seconds: 60, step_seconds: 20, phase_seconds: 5, + ttl_seconds: None, }; assert!(phased.is_bucket_start(5_000)); assert!(phased.is_bucket_start(45_000)); @@ -494,6 +548,7 @@ mod tests { range_seconds: 60, step_seconds: 0, phase_seconds: 0, + ttl_seconds: None, }; assert!(!malformed.is_bucket_start(0)); } @@ -505,6 +560,7 @@ mod tests { range_seconds: 60, step_seconds: 20, phase_seconds: 5, + ttl_seconds: None, }; // starts are the 5th, 25th, 45th, ... second; now = the 50th second → // most recent start is the 45th @@ -525,6 +581,7 @@ mod tests { range_seconds: 60, step_seconds: 20, phase_seconds: 5, + ttl_seconds: None, }; assert_eq!(t.storage_key("$createdAt"), "$createdAt#60#20#5"); // two grids over the same property produce distinct sibling keys — @@ -535,12 +592,14 @@ mod tests { range_seconds: 6 * HOUR_SECONDS, step_seconds: 6 * HOUR_SECONDS, phase_seconds: 0, + ttl_seconds: None, }; let three_hourly = TimeRangeTransform { source: "$createdAt".to_string(), range_seconds: 3 * HOUR_SECONDS, step_seconds: 3 * HOUR_SECONDS, phase_seconds: 0, + ttl_seconds: None, }; assert_ne!( six_hourly.storage_key("$createdAt"), diff --git a/packages/rs-drive-abci/Cargo.toml b/packages/rs-drive-abci/Cargo.toml index 402c94b574e..2f4bb897d05 100644 --- a/packages/rs-drive-abci/Cargo.toml +++ b/packages/rs-drive-abci/Cargo.toml @@ -82,7 +82,7 @@ derive_more = { version = "1.0", features = ["from", "deref", "deref_mut"] } async-trait = "0.1.77" console-subscriber = { version = "0.4", optional = true } bls-signatures = { git = "https://github.com/dashpay/bls-signatures", rev = "0842b17583888e8f46c252a4ee84cdfd58e0546f", optional = true } -grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "97250247423ddc7fcc2865ea0025fc5bd41d0883" } +grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "d548e28228e2aca1361d9b12835388be0f811a92" } nonempty = "0.11" # Shielded-pool snapshot needs raw RocksDB SstFileWriter + ingest_external_file_cf # bindings, and blake3 for the snapshot-file checksum. @@ -108,7 +108,7 @@ dpp = { path = "../rs-dpp", default-features = false, features = [ drive = { path = "../rs-drive", features = ["fixtures-and-mocks"] } drive-proof-verifier = { path = "../rs-drive-proof-verifier" } strategy-tests = { path = "../strategy-tests" } -grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "97250247423ddc7fcc2865ea0025fc5bd41d0883", features = ["client"] } +grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "d548e28228e2aca1361d9b12835388be0f811a92", features = ["client"] } assert_matches = "1.5.0" drive-abci = { path = ".", features = ["testing-config", "mocks", "shielded_test_data"] } bls-signatures = { git = "https://github.com/dashpay/bls-signatures", rev = "0842b17583888e8f46c252a4ee84cdfd58e0546f" } @@ -122,8 +122,8 @@ integer-encoding = { version = "4.0.0" } # For dump_only_default_and_aux_cfs_under_shielded_subtree_prefix — same # subtree-prefix algorithm grovedb uses internally. -grovedb-path = { git = "https://github.com/dashpay/grovedb", rev = "97250247423ddc7fcc2865ea0025fc5bd41d0883" } -grovedb-storage = { git = "https://github.com/dashpay/grovedb", rev = "97250247423ddc7fcc2865ea0025fc5bd41d0883" } +grovedb-path = { git = "https://github.com/dashpay/grovedb", rev = "d548e28228e2aca1361d9b12835388be0f811a92" } +grovedb-storage = { git = "https://github.com/dashpay/grovedb", rev = "d548e28228e2aca1361d9b12835388be0f811a92" } [features] default = ["bls-signatures"] diff --git a/packages/rs-drive-abci/src/abci/handler/finalize_block.rs b/packages/rs-drive-abci/src/abci/handler/finalize_block.rs index ade56bf1135..f79404a14b7 100644 --- a/packages/rs-drive-abci/src/abci/handler/finalize_block.rs +++ b/packages/rs-drive-abci/src/abci/handler/finalize_block.rs @@ -96,6 +96,36 @@ where .committed_block_height_guard .store(block_height, Ordering::Relaxed); + // Reclaim the storage of any TTL'd time-range buckets the block's + // transaction flat-dropped (grovedb#848 / PR #849): the drops' redo + // records became visible at the commit above, and the flush turns them + // into DB-level range tombstones. Outside consensus by design — it + // never touches the root hash, the report is telemetry, and a failure + // leaves the records in place for the next flush (or the startup + // flush) to retry, so the finalized block is unaffected. + match app + .platform() + .drive + .grove + .flush_pending_prefix_drops(&platform_version.drive.grove_version) + { + Ok(report) => { + if report.reclaimed_records > 0 || report.skipped_live > 0 { + tracing::debug!( + reclaimed_records = report.reclaimed_records, + skipped_live = report.skipped_live, + "flushed pending prefix drops" + ); + } + } + Err(error) => { + tracing::warn!( + ?error, + "failed to flush pending prefix drops; records persist and will be retried" + ); + } + } + // Create GroveDB checkpoint after the transaction is committed (so it captures committed state) if block_finalization_outcome.checkpoint_needed { app.platform().create_grovedb_checkpoint(platform_version)?; diff --git a/packages/rs-drive-abci/src/platform_types/platform/mod.rs b/packages/rs-drive-abci/src/platform_types/platform/mod.rs index 972e530a1ce..b6dba09760e 100644 --- a/packages/rs-drive-abci/src/platform_types/platform/mod.rs +++ b/packages/rs-drive-abci/src/platform_types/platform/mod.rs @@ -147,6 +147,24 @@ impl Platform { let (drive, current_platform_version) = Drive::open(&config.db_path, Some(config.drive.clone())).map_err(Error::Drive)?; + // Finish any TTL bucket-drop reclamation a crash interrupted + // (grovedb#848 / PR #849): committed redo records survive restarts, + // and draining them is idempotent and outside consensus. A no-op + // when no records exist; a failure leaves the records for the + // per-block flush to retry. + if let Some(platform_version) = current_platform_version { + if let Err(error) = drive + .grove + .flush_pending_prefix_drops(&platform_version.drive.grove_version) + { + tracing::warn!( + ?error, + "failed to flush pending prefix drops at startup; records persist and \ + will be retried after the next block" + ); + } + } + if let Some(platform_version) = current_platform_version { let Some(execution_state) = Platform::::fetch_platform_state(&drive, None, platform_version)? diff --git a/packages/rs-drive-abci/src/query/document_query/v1/tests.rs b/packages/rs-drive-abci/src/query/document_query/v1/tests.rs index 10fd4fe471d..5f07343f43d 100644 --- a/packages/rs-drive-abci/src/query/document_query/v1/tests.rs +++ b/packages/rs-drive-abci/src/query/document_query/v1/tests.rs @@ -5605,9 +5605,9 @@ mod time_range_proof_verification { /// The mirror image of the relative selectors' tamper tests: a /// `BY_START` proof is clock-invariant. Re-signing the response over a /// one-step-later metadata time still verifies, because the window is - /// named absolutely in the query and its resolution never consults the - /// signed time — the property that makes historic windows stable to - /// query. (The same nudge makes a `newest` proof fail; see + /// named absolutely in the query and its resolution consults the + /// signed time only for the TTL horizon gate (no `ttl` here, so not at + /// all) — the property that makes historic windows stable to query. (The same nudge makes a `newest` proof fail; see /// [`a_tampered_metadata_time_is_rejected_at_the_sum_entry_point`].) #[test] fn a_by_start_proof_is_indifferent_to_the_signed_time() { diff --git a/packages/rs-drive/Cargo.toml b/packages/rs-drive/Cargo.toml index 365b1de692f..86dbfd5f36b 100644 --- a/packages/rs-drive/Cargo.toml +++ b/packages/rs-drive/Cargo.toml @@ -52,13 +52,13 @@ enum-map = { version = "2.0.3", optional = true } intmap = { version = "3.0.1", features = ["serde"], optional = true } chrono = { version = "0.4.35", optional = true } itertools = { version = "0.13", optional = true } -grovedb = { git = "https://github.com/dashpay/grovedb", rev = "97250247423ddc7fcc2865ea0025fc5bd41d0883", optional = true, default-features = false } -grovedb-costs = { git = "https://github.com/dashpay/grovedb", rev = "97250247423ddc7fcc2865ea0025fc5bd41d0883", optional = true } -grovedb-path = { git = "https://github.com/dashpay/grovedb", rev = "97250247423ddc7fcc2865ea0025fc5bd41d0883" } -grovedb-query = { git = "https://github.com/dashpay/grovedb", rev = "97250247423ddc7fcc2865ea0025fc5bd41d0883" } -grovedb-storage = { git = "https://github.com/dashpay/grovedb", rev = "97250247423ddc7fcc2865ea0025fc5bd41d0883", optional = true } -grovedb-version = { git = "https://github.com/dashpay/grovedb", rev = "97250247423ddc7fcc2865ea0025fc5bd41d0883" } -grovedb-epoch-based-storage-flags = { git = "https://github.com/dashpay/grovedb", rev = "97250247423ddc7fcc2865ea0025fc5bd41d0883" } +grovedb = { git = "https://github.com/dashpay/grovedb", rev = "d548e28228e2aca1361d9b12835388be0f811a92", optional = true, default-features = false } +grovedb-costs = { git = "https://github.com/dashpay/grovedb", rev = "d548e28228e2aca1361d9b12835388be0f811a92", optional = true } +grovedb-path = { git = "https://github.com/dashpay/grovedb", rev = "d548e28228e2aca1361d9b12835388be0f811a92" } +grovedb-query = { git = "https://github.com/dashpay/grovedb", rev = "d548e28228e2aca1361d9b12835388be0f811a92" } +grovedb-storage = { git = "https://github.com/dashpay/grovedb", rev = "d548e28228e2aca1361d9b12835388be0f811a92", optional = true } +grovedb-version = { git = "https://github.com/dashpay/grovedb", rev = "d548e28228e2aca1361d9b12835388be0f811a92" } +grovedb-epoch-based-storage-flags = { git = "https://github.com/dashpay/grovedb", rev = "d548e28228e2aca1361d9b12835388be0f811a92" } [dev-dependencies] criterion = "0.5" diff --git a/packages/rs-drive/src/drive/contract/insert/insert_contract/v0/tests/index_only_e2e_tests.rs b/packages/rs-drive/src/drive/contract/insert/insert_contract/v0/tests/index_only_e2e_tests.rs index 960fdd4347c..7be1c2aa778 100644 --- a/packages/rs-drive/src/drive/contract/insert/insert_contract/v0/tests/index_only_e2e_tests.rs +++ b/packages/rs-drive/src/drive/contract/insert/insert_contract/v0/tests/index_only_e2e_tests.rs @@ -2260,6 +2260,7 @@ fn beat_bucket_counts_serve_trending() { range_seconds: 3600, step_seconds: 900, phase_seconds: 0, + ttl_seconds: None, }; let resolved = vec![ResolvedTimeRange { transform: transform.clone(), @@ -2402,6 +2403,7 @@ fn beat_synthesis_over_bucketed_index_is_refused() { range_seconds: 3600, step_seconds: 900, phase_seconds: 0, + ttl_seconds: None, }, }], }; diff --git a/packages/rs-drive/src/drive/contract/update/update_keywords/v0/mod.rs b/packages/rs-drive/src/drive/contract/update/update_keywords/v0/mod.rs index 15a847c7e8a..2c65c180887 100644 --- a/packages/rs-drive/src/drive/contract/update/update_keywords/v0/mod.rs +++ b/packages/rs-drive/src/drive/contract/update/update_keywords/v0/mod.rs @@ -148,6 +148,7 @@ impl Drive { document_type, None, estimated_costs_only_with_layer_info, + block_info.time_ms, transaction, platform_version, )?); diff --git a/packages/rs-drive/src/drive/document/delete/delete_document_for_contract/v0/mod.rs b/packages/rs-drive/src/drive/document/delete/delete_document_for_contract/v0/mod.rs index 86505ec6a03..b82d4b25c51 100644 --- a/packages/rs-drive/src/drive/document/delete/delete_document_for_contract/v0/mod.rs +++ b/packages/rs-drive/src/drive/document/delete/delete_document_for_contract/v0/mod.rs @@ -38,6 +38,7 @@ impl Drive { contract, document_type_name, estimated_costs_only_with_layer_info, + block_info.time_ms, transaction, &mut drive_operations, platform_version, diff --git a/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_apply_and_add_to_operations/mod.rs b/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_apply_and_add_to_operations/mod.rs index 5aef315d8f7..e1a451d52a1 100644 --- a/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_apply_and_add_to_operations/mod.rs +++ b/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_apply_and_add_to_operations/mod.rs @@ -37,6 +37,7 @@ impl Drive { estimated_costs_only_with_layer_info: Option< HashMap, >, + block_time_ms: u64, transaction: TransactionArg, drive_operations: &mut Vec, platform_version: &PlatformVersion, @@ -53,6 +54,7 @@ impl Drive { contract, document_type_name, estimated_costs_only_with_layer_info, + block_time_ms, transaction, drive_operations, platform_version, diff --git a/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_apply_and_add_to_operations/v0/mod.rs b/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_apply_and_add_to_operations/v0/mod.rs index db151c9e972..2913e7d4c5c 100644 --- a/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_apply_and_add_to_operations/v0/mod.rs +++ b/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_apply_and_add_to_operations/v0/mod.rs @@ -21,6 +21,7 @@ impl Drive { mut estimated_costs_only_with_layer_info: Option< HashMap, >, + block_time_ms: u64, transaction: TransactionArg, drive_operations: &mut Vec, platform_version: &PlatformVersion, @@ -31,6 +32,7 @@ impl Drive { document_type_name, None, &mut estimated_costs_only_with_layer_info, + block_time_ms, transaction, platform_version, )?; diff --git a/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_id/v0/mod.rs b/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_id/v0/mod.rs index 70e4e5716e6..50e415bbb06 100644 --- a/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_id/v0/mod.rs +++ b/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_id/v0/mod.rs @@ -57,6 +57,7 @@ impl Drive { contract, document_type_name, estimated_costs_only_with_layer_info, + block_info.time_ms, transaction, &mut drive_operations, platform_version, diff --git a/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_id_with_named_type_operations/mod.rs b/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_id_with_named_type_operations/mod.rs index 5e8efed9771..10d1297bdf0 100644 --- a/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_id_with_named_type_operations/mod.rs +++ b/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_id_with_named_type_operations/mod.rs @@ -40,6 +40,7 @@ impl Drive { estimated_costs_only_with_layer_info: &mut Option< HashMap, >, + block_time_ms: u64, transaction: TransactionArg, platform_version: &PlatformVersion, ) -> Result, Error> { @@ -57,6 +58,7 @@ impl Drive { epoch, previous_batch_operations, estimated_costs_only_with_layer_info, + block_time_ms, transaction, platform_version, ), diff --git a/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_id_with_named_type_operations/v0/mod.rs b/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_id_with_named_type_operations/v0/mod.rs index 85d082fd4a3..1349e7e07a6 100644 --- a/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_id_with_named_type_operations/v0/mod.rs +++ b/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_id_with_named_type_operations/v0/mod.rs @@ -30,6 +30,7 @@ impl Drive { estimated_costs_only_with_layer_info: &mut Option< HashMap, >, + block_time_ms: u64, transaction: TransactionArg, platform_version: &PlatformVersion, ) -> Result, Error> { @@ -54,6 +55,7 @@ impl Drive { document_type, previous_batch_operations, estimated_costs_only_with_layer_info, + block_time_ms, transaction, platform_version, ) diff --git a/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_operations/mod.rs b/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_operations/mod.rs index 74fefe304d9..0f5fbab5237 100644 --- a/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_operations/mod.rs +++ b/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_operations/mod.rs @@ -39,6 +39,7 @@ impl Drive { estimated_costs_only_with_layer_info: &mut Option< HashMap, >, + block_time_ms: u64, transaction: TransactionArg, platform_version: &PlatformVersion, ) -> Result, Error> { @@ -55,6 +56,7 @@ impl Drive { document_type, previous_batch_operations, estimated_costs_only_with_layer_info, + block_time_ms, transaction, platform_version, ), @@ -91,6 +93,7 @@ impl Drive { estimated_costs_only_with_layer_info: &mut Option< HashMap, >, + block_time_ms: u64, transaction: TransactionArg, platform_version: &PlatformVersion, ) -> Result, Error> { @@ -107,6 +110,7 @@ impl Drive { document_type, previous_batch_operations, estimated_costs_only_with_layer_info, + block_time_ms, transaction, platform_version, ), diff --git a/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_operations/v0/mod.rs b/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_operations/v0/mod.rs index 9b9ab080a37..0134b878d25 100644 --- a/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_operations/v0/mod.rs +++ b/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_operations/v0/mod.rs @@ -47,6 +47,7 @@ impl Drive { estimated_costs_only_with_layer_info: &mut Option< HashMap, >, + block_time_ms: u64, transaction: TransactionArg, platform_version: &PlatformVersion, ) -> Result, Error> { @@ -62,6 +63,7 @@ impl Drive { document_type, previous_batch_operations, estimated_costs_only_with_layer_info, + block_time_ms, transaction, platform_version, ) @@ -79,6 +81,7 @@ impl Drive { estimated_costs_only_with_layer_info: &mut Option< HashMap, >, + block_time_ms: u64, transaction: TransactionArg, platform_version: &PlatformVersion, ) -> Result, Error> { @@ -198,6 +201,7 @@ impl Drive { &document_and_contract_info, &previous_batch_operations, estimated_costs_only_with_layer_info, + block_time_ms, transaction, &mut batch_operations, platform_version, diff --git a/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_with_named_type_operations/mod.rs b/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_with_named_type_operations/mod.rs index 5c57a29a1a1..264aaaf7a97 100644 --- a/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_with_named_type_operations/mod.rs +++ b/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_with_named_type_operations/mod.rs @@ -38,6 +38,7 @@ impl Drive { estimated_costs_only_with_layer_info: &mut Option< HashMap, >, + block_time_ms: u64, transaction: TransactionArg, platform_version: &PlatformVersion, ) -> Result, Error> { @@ -54,6 +55,7 @@ impl Drive { document_type_name, previous_batch_operations, estimated_costs_only_with_layer_info, + block_time_ms, transaction, platform_version, ), diff --git a/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_with_named_type_operations/v0/mod.rs b/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_with_named_type_operations/v0/mod.rs index f7babbe7f7c..cddde01f1b0 100644 --- a/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_with_named_type_operations/v0/mod.rs +++ b/packages/rs-drive/src/drive/document/delete/delete_document_for_contract_with_named_type_operations/v0/mod.rs @@ -29,6 +29,7 @@ impl Drive { estimated_costs_only_with_layer_info: &mut Option< HashMap, >, + block_time_ms: u64, transaction: TransactionArg, platform_version: &PlatformVersion, ) -> Result, Error> { @@ -39,6 +40,7 @@ impl Drive { document_type, previous_batch_operations, estimated_costs_only_with_layer_info, + block_time_ms, transaction, platform_version, ) diff --git a/packages/rs-drive/src/drive/document/delete/delete_index_only_document_for_contract_operations/mod.rs b/packages/rs-drive/src/drive/document/delete/delete_index_only_document_for_contract_operations/mod.rs index fe4c43ec72b..6503bc7c7f5 100644 --- a/packages/rs-drive/src/drive/document/delete/delete_index_only_document_for_contract_operations/mod.rs +++ b/packages/rs-drive/src/drive/document/delete/delete_index_only_document_for_contract_operations/mod.rs @@ -40,6 +40,7 @@ impl Drive { estimated_costs_only_with_layer_info: &mut Option< HashMap, >, + block_time_ms: u64, transaction: TransactionArg, platform_version: &PlatformVersion, ) -> Result, Error> { @@ -56,6 +57,7 @@ impl Drive { document_type, previous_batch_operations, estimated_costs_only_with_layer_info, + block_time_ms, transaction, platform_version, ), diff --git a/packages/rs-drive/src/drive/document/delete/delete_index_only_document_for_contract_operations/v0/mod.rs b/packages/rs-drive/src/drive/document/delete/delete_index_only_document_for_contract_operations/v0/mod.rs index 3b654703ac9..cb3898624d3 100644 --- a/packages/rs-drive/src/drive/document/delete/delete_index_only_document_for_contract_operations/v0/mod.rs +++ b/packages/rs-drive/src/drive/document/delete/delete_index_only_document_for_contract_operations/v0/mod.rs @@ -61,6 +61,7 @@ impl Drive { document_type, None, &mut estimated_costs_only_with_layer_info, + block_info.time_ms, transaction, platform_version, )?; @@ -103,6 +104,7 @@ impl Drive { estimated_costs_only_with_layer_info: &mut Option< HashMap, >, + block_time_ms: u64, transaction: TransactionArg, platform_version: &PlatformVersion, ) -> Result, Error> { @@ -184,6 +186,7 @@ impl Drive { &document_and_contract_info, &previous_batch_operations, estimated_costs_only_with_layer_info, + block_time_ms, transaction, &mut batch_operations, platform_version, diff --git a/packages/rs-drive/src/drive/document/delete/mod.rs b/packages/rs-drive/src/drive/document/delete/mod.rs index 03cd835a01e..e04dc6eb7f7 100644 --- a/packages/rs-drive/src/drive/document/delete/mod.rs +++ b/packages/rs-drive/src/drive/document/delete/mod.rs @@ -1445,6 +1445,7 @@ mod tests { &epoch, None, &mut estimated_costs_only_with_layer_info, + 0, None, platform_version, ) diff --git a/packages/rs-drive/src/drive/document/delete/remove_indices_for_index_level_for_contract_operations/mod.rs b/packages/rs-drive/src/drive/document/delete/remove_indices_for_index_level_for_contract_operations/mod.rs index 85ddd3b9b51..9dcc89b88ca 100644 --- a/packages/rs-drive/src/drive/document/delete/remove_indices_for_index_level_for_contract_operations/mod.rs +++ b/packages/rs-drive/src/drive/document/delete/remove_indices_for_index_level_for_contract_operations/mod.rs @@ -59,6 +59,7 @@ impl Drive { estimated_costs_only_with_layer_info: &mut Option< HashMap, >, + skip_missing_expired_entry: bool, event_id: [u8; 32], transaction: TransactionArg, batch_operations: &mut Vec, @@ -120,6 +121,7 @@ impl Drive { storage_flags, previous_batch_operations, estimated_costs_only_with_layer_info, + skip_missing_expired_entry, event_id, transaction, batch_operations, diff --git a/packages/rs-drive/src/drive/document/delete/remove_indices_for_index_level_for_contract_operations/v0/mod.rs b/packages/rs-drive/src/drive/document/delete/remove_indices_for_index_level_for_contract_operations/v0/mod.rs index 9da9dae15cf..de5263542e2 100644 --- a/packages/rs-drive/src/drive/document/delete/remove_indices_for_index_level_for_contract_operations/v0/mod.rs +++ b/packages/rs-drive/src/drive/document/delete/remove_indices_for_index_level_for_contract_operations/v0/mod.rs @@ -87,6 +87,7 @@ impl Drive { storage_flags, previous_batch_operations, estimated_costs_only_with_layer_info, + false, event_id, transaction, batch_operations, diff --git a/packages/rs-drive/src/drive/document/delete/remove_indices_for_index_level_for_contract_operations/v1/mod.rs b/packages/rs-drive/src/drive/document/delete/remove_indices_for_index_level_for_contract_operations/v1/mod.rs index 924722b934c..18c8a7ce2fc 100644 --- a/packages/rs-drive/src/drive/document/delete/remove_indices_for_index_level_for_contract_operations/v1/mod.rs +++ b/packages/rs-drive/src/drive/document/delete/remove_indices_for_index_level_for_contract_operations/v1/mod.rs @@ -93,6 +93,7 @@ impl Drive { storage_flags, previous_batch_operations, estimated_costs_only_with_layer_info, + false, event_id, transaction, batch_operations, diff --git a/packages/rs-drive/src/drive/document/delete/remove_indices_for_index_level_for_contract_operations/v2/mod.rs b/packages/rs-drive/src/drive/document/delete/remove_indices_for_index_level_for_contract_operations/v2/mod.rs index a26cf2b9047..ac9986bd71d 100644 --- a/packages/rs-drive/src/drive/document/delete/remove_indices_for_index_level_for_contract_operations/v2/mod.rs +++ b/packages/rs-drive/src/drive/document/delete/remove_indices_for_index_level_for_contract_operations/v2/mod.rs @@ -58,6 +58,7 @@ impl Drive { estimated_costs_only_with_layer_info: &mut Option< HashMap, >, + skip_missing_expired_entry: bool, event_id: [u8; 32], transaction: TransactionArg, batch_operations: &mut Vec, @@ -93,6 +94,7 @@ impl Drive { storage_flags, previous_batch_operations, estimated_costs_only_with_layer_info, + skip_missing_expired_entry, event_id, transaction, batch_operations, @@ -180,6 +182,7 @@ impl Drive { storage_flags, previous_batch_operations, estimated_costs_only_with_layer_info, + skip_missing_expired_entry, event_id, transaction, batch_operations, diff --git a/packages/rs-drive/src/drive/document/delete/remove_indices_for_top_index_level_for_contract_operations/mod.rs b/packages/rs-drive/src/drive/document/delete/remove_indices_for_top_index_level_for_contract_operations/mod.rs index 65e0d8bd98e..f9b42a58cc8 100644 --- a/packages/rs-drive/src/drive/document/delete/remove_indices_for_top_index_level_for_contract_operations/mod.rs +++ b/packages/rs-drive/src/drive/document/delete/remove_indices_for_top_index_level_for_contract_operations/mod.rs @@ -31,6 +31,7 @@ impl Drive { /// # Returns /// * `Ok(())` if the operation was successful. /// * `Err(DriveError::UnknownVersionMismatch)` if the drive version does not match known versions. + #[allow(clippy::too_many_arguments)] pub(super) fn remove_indices_for_top_index_level_for_contract_operations( &self, document_and_contract_info: &DocumentAndContractInfo, @@ -38,6 +39,7 @@ impl Drive { estimated_costs_only_with_layer_info: &mut Option< HashMap, >, + block_time_ms: u64, transaction: TransactionArg, batch_operations: &mut Vec, platform_version: &PlatformVersion, @@ -71,6 +73,7 @@ impl Drive { document_and_contract_info, previous_batch_operations, estimated_costs_only_with_layer_info, + block_time_ms, transaction, batch_operations, platform_version, diff --git a/packages/rs-drive/src/drive/document/delete/remove_indices_for_top_index_level_for_contract_operations/v0/mod.rs b/packages/rs-drive/src/drive/document/delete/remove_indices_for_top_index_level_for_contract_operations/v0/mod.rs index ef129d79a32..67d2ad620a4 100644 --- a/packages/rs-drive/src/drive/document/delete/remove_indices_for_top_index_level_for_contract_operations/v0/mod.rs +++ b/packages/rs-drive/src/drive/document/delete/remove_indices_for_top_index_level_for_contract_operations/v0/mod.rs @@ -181,6 +181,7 @@ impl Drive { &storage_flags, previous_batch_operations, estimated_costs_only_with_layer_info, + false, event_id, transaction, batch_operations, diff --git a/packages/rs-drive/src/drive/document/delete/remove_indices_for_top_index_level_for_contract_operations/v1/mod.rs b/packages/rs-drive/src/drive/document/delete/remove_indices_for_top_index_level_for_contract_operations/v1/mod.rs index d1e74ecd39a..e0c64f5906a 100644 --- a/packages/rs-drive/src/drive/document/delete/remove_indices_for_top_index_level_for_contract_operations/v1/mod.rs +++ b/packages/rs-drive/src/drive/document/delete/remove_indices_for_top_index_level_for_contract_operations/v1/mod.rs @@ -212,6 +212,7 @@ impl Drive { &storage_flags, previous_batch_operations, estimated_costs_only_with_layer_info, + false, event_id, transaction, batch_operations, diff --git a/packages/rs-drive/src/drive/document/delete/remove_indices_for_top_index_level_for_contract_operations/v2/mod.rs b/packages/rs-drive/src/drive/document/delete/remove_indices_for_top_index_level_for_contract_operations/v2/mod.rs index 9ae6c9cfd5d..f2fd88d5bfe 100644 --- a/packages/rs-drive/src/drive/document/delete/remove_indices_for_top_index_level_for_contract_operations/v2/mod.rs +++ b/packages/rs-drive/src/drive/document/delete/remove_indices_for_top_index_level_for_contract_operations/v2/mod.rs @@ -11,6 +11,7 @@ use crate::drive::document::estimation_costs::estimated_sum_trees_for_value_tree use crate::drive::document::index_level_tree_types::{ index_level_tree_types_with_continuation_demotion, time_range_index_keys, }; +use crate::drive::document::time_range_ttl::entry_key_bucket_start; use crate::drive::document::unique_event_id; use crate::util::type_constants::DEFAULT_HASH_SIZE_U8; @@ -42,6 +43,7 @@ impl Drive { /// [`Drive::add_indices_for_top_index_level_for_contract_operations_v2`]; /// part of the platform v14 shared-prefix aggregate fix. #[inline(always)] + #[allow(clippy::too_many_arguments)] pub(super) fn remove_indices_for_top_index_level_for_contract_operations_v2( &self, document_and_contract_info: &DocumentAndContractInfo, @@ -49,6 +51,7 @@ impl Drive { estimated_costs_only_with_layer_info: &mut Option< HashMap, >, + block_time_ms: u64, transaction: TransactionArg, batch_operations: &mut Vec, platform_version: &PlatformVersion, @@ -85,6 +88,26 @@ impl Drive { document_and_contract_info.document_type.name().as_str(), ); + // TTL drainage rides every write into a TTL'd index — deletes + // included: without this, an index receiving only deletions would + // never advance cleanup, breaking the documented every-write rule. + // One sweep over the deduplicated levels, BEFORE any delete + // mutation is queued (drainage applies directly to grovedb, so a + // later drain could remove a path a queued operation targets), and + // before the expired/standing detection below so it sees post-drain + // state. Stateful only — the estimation dry run neither reads state + // nor prices drops. Unbilled — see the ttl module's Billing + // section. + if estimated_costs_only_with_layer_info.is_none() { + self.drain_expired_time_range_levels( + index_level, + &contract_document_type_path, + block_time_ms, + transaction, + platform_version, + )?; + } + let sub_level_index_count = index_level.sub_levels().len() as u32; if let Some(estimated_costs_only_with_layer_info) = estimated_costs_only_with_layer_info { @@ -114,6 +137,20 @@ impl Drive { let property_name_tree_type = tree_types.property_name_tree_type; let value_tree_type = tree_types.value_tree_type; + // Mirror of the insert walker's ephemeral routing: removals + // under a TTL'd sub-level ride the ephemeral batch and are + // consumed at the ephemeral price — their elements carry no + // storage flags, so removal is basic and yields no refunds. + let sub_level_is_ephemeral = sub_level + .time_range() + .is_some_and(|transform| transform.ttl_seconds.is_some()); + let mut ephemeral_local_operations: Vec = vec![]; + let index_storage_flags = if sub_level_is_ephemeral { + None + } else { + storage_flags + }; + // at this point the contract path is to the contract documents // for each index the top index component will already have been added // when the contract itself was created @@ -190,7 +227,7 @@ impl Drive { estimated_layer_sizes: AllSubtrees( document_top_field_estimated_size as u8, estimated_sum_trees_for_value_tree_type(value_tree_type), - storage_flags.map(|s| s.serialized_size()), + index_storage_flags.map(|s| s.serialized_size()), ), }, ); @@ -221,6 +258,43 @@ impl Drive { let bucket_count = index_keys.len(); for (bucket, index_key) in index_keys.into_iter().enumerate() { + // TTL: an expired bucket may already have been dropped + // entirely (this document's entries went with it — skip), + // or stand PARTIALLY drained (drainage removes whole `[0]` + // and group value trees before the bucket): removal then + // proceeds, but at full-path granularity — the deeper + // walkers skip any entry whose path the drain already + // took. Live buckets behave exactly as before. Stateful + // reads have no place in the estimation dry run, which + // processes every bucket — the upper bound. + let mut skip_missing_expired_entry = false; + if estimated_costs_only_with_layer_info.is_none() { + if let Some(transform) = sub_level.time_range() { + let entry_key_bytes = match &index_key { + DriveKeyInfo::Key(key) => Some(key.as_slice()), + DriveKeyInfo::KeyRef(key) => Some(*key), + DriveKeyInfo::KeySize(_) => None, + }; + if let Some(entry_key_bytes) = entry_key_bytes { + let expired = entry_key_bucket_start(entry_key_bytes) + .zip(transform.expiry_horizon_ms(block_time_ms)) + .is_some_and(|(start, horizon)| start < horizon); + if expired { + if !self.time_range_entry_is_removable( + transform, + entry_key_bytes, + block_time_ms, + &index_path, + transaction, + platform_version, + )? { + continue; + } + skip_missing_expired_entry = true; + } + } + } + } // The final bucket takes ownership of `index_path`; earlier // buckets (only a time-range fan-out has more than one) // clone it. @@ -244,6 +318,12 @@ impl Drive { index_path_info.push(index_key)?; // the index path is now something likeDataContracts/ContractID/Documents(1)/$ownerId/ + let index_batch_operations: &mut Vec = + if sub_level_is_ephemeral { + &mut ephemeral_local_operations + } else { + &mut *batch_operations + }; self.remove_indices_for_index_level_for_contract_operations( document_and_contract_info, index_path_info, @@ -251,15 +331,24 @@ impl Drive { any_fields_null, all_fields_null, value_tree_type, - &storage_flags, + &index_storage_flags, previous_batch_operations, estimated_costs_only_with_layer_info, + skip_missing_expired_entry, event_id, transaction, - batch_operations, + index_batch_operations, platform_version, )?; } + + if sub_level_is_ephemeral { + batch_operations.extend( + ephemeral_local_operations + .into_iter() + .map(LowLevelDriveOperation::retag_ephemeral), + ); + } } Ok(()) } diff --git a/packages/rs-drive/src/drive/document/delete/remove_reference_for_index_level_for_contract_operations/mod.rs b/packages/rs-drive/src/drive/document/delete/remove_reference_for_index_level_for_contract_operations/mod.rs index 4e7fcd885dc..e99130c4c72 100644 --- a/packages/rs-drive/src/drive/document/delete/remove_reference_for_index_level_for_contract_operations/mod.rs +++ b/packages/rs-drive/src/drive/document/delete/remove_reference_for_index_level_for_contract_operations/mod.rs @@ -51,6 +51,7 @@ impl Drive { estimated_costs_only_with_layer_info: &mut Option< HashMap, >, + skip_missing_expired_entry: bool, event_id: [u8; 32], transaction: TransactionArg, batch_operations: &mut Vec, @@ -88,6 +89,7 @@ impl Drive { storage_flags, previous_batch_operations, estimated_costs_only_with_layer_info, + skip_missing_expired_entry, event_id, transaction, batch_operations, diff --git a/packages/rs-drive/src/drive/document/delete/remove_reference_for_index_level_for_contract_operations/v1/mod.rs b/packages/rs-drive/src/drive/document/delete/remove_reference_for_index_level_for_contract_operations/v1/mod.rs index a48f9d7b02e..5f5b2ea2447 100644 --- a/packages/rs-drive/src/drive/document/delete/remove_reference_for_index_level_for_contract_operations/v1/mod.rs +++ b/packages/rs-drive/src/drive/document/delete/remove_reference_for_index_level_for_contract_operations/v1/mod.rs @@ -53,6 +53,7 @@ impl Drive { estimated_costs_only_with_layer_info: &mut Option< HashMap, >, + skip_missing_expired_entry: bool, event_id: [u8; 32], transaction: TransactionArg, batch_operations: &mut Vec, @@ -63,6 +64,57 @@ impl Drive { } let mut key_info_path = index_path_info.convert_to_key_info_path(); + // TTL: for an entry in an expired-but-standing bucket, drainage may + // already have removed the deeper trees (the group's `[0]` and value + // tree go before the bucket does). The entry path is checked at + // full granularity and the whole reference removal skipped when it + // is gone — the drain took it, and there is nothing left to + // remove. Only set on the stateful path, so every KeyInfo below is + // a KnownKey. + if skip_missing_expired_entry { + let mut path_segments: Vec> = key_info_path + .0 + .iter() + .map(|key_info| match key_info { + KnownKey(key) => Ok(key.clone()), + _ => Err(Error::Drive( + crate::error::drive::DriveError::CorruptedCodeExecution( + "expired-entry skip is stateful-only; its path must be known", + ), + )), + }) + .collect::>()?; + // The layouts that store the reference inside a `[0]` subtree + // must include that segment in the walk: the drain drops the + // flat `[0]` tree BEFORE its value tree, and a budget boundary + // between the two leaves the value tree standing with the + // `[0]` gone — every shallower segment then exists while the + // delete below would target the missing subtree. Mirrors the + // layout dispatch used for the actual delete: the indexOnly + // terminal and the non-unique / contested / null layouts use + // `[0]` trees; the unique layout stores the reference AT key + // `[0]` as a bare element of the value tree. + let uses_zero_subtree = index_type.terminal.is_some() + || index_type.index_type == NonUniqueIndex + || index_type.index_type == ContestedResourceIndex + || any_fields_null; + if uses_zero_subtree { + path_segments.push(vec![0]); + } + // The document-type path plus the (grid-qualified) index level + // key exist for every registered contract: the flag is only + // ever set for a bucketed index, whose level tree is created at + // contract registration — so the walk starts below them. + if !self.expired_entry_path_exists( + &path_segments, + usize::from(CONTRACT_DOCUMENTS_PATH_HEIGHT) + 1, + transaction, + platform_version, + )? { + return Ok(()); + } + } + let document_type = document_and_contract_info.document_type; // indexOnly terminal: the member key is the terminal property's diff --git a/packages/rs-drive/src/drive/document/index_level_tree_types.rs b/packages/rs-drive/src/drive/document/index_level_tree_types.rs index c9158c654e1..05860ddd970 100644 --- a/packages/rs-drive/src/drive/document/index_level_tree_types.rs +++ b/packages/rs-drive/src/drive/document/index_level_tree_types.rs @@ -822,6 +822,7 @@ mod tests { range_seconds: 21_600, step_seconds: 7_200, phase_seconds: 0, + ttl_seconds: None, }; let key = DriveKeyInfo::KeySize(KeyInfo::MaxKeySize { unique_id: vec![7u8; 4], @@ -860,6 +861,7 @@ mod tests { range_seconds: 100 * 3_600, step_seconds: 3_600, phase_seconds: 0, + ttl_seconds: None, }; assert_eq!(oversized.overlap_factor(), 100); let keys = time_range_index_keys(Some(&oversized), key, 24); diff --git a/packages/rs-drive/src/drive/document/insert/add_document_for_contract/time_range_index_e2e_tests.rs b/packages/rs-drive/src/drive/document/insert/add_document_for_contract/time_range_index_e2e_tests.rs index e8e60a5dabf..2ffbf23de30 100644 --- a/packages/rs-drive/src/drive/document/insert/add_document_for_contract/time_range_index_e2e_tests.rs +++ b/packages/rs-drive/src/drive/document/insert/add_document_for_contract/time_range_index_e2e_tests.rs @@ -27,9 +27,11 @@ use dpp::data_contract::document_type::DocumentTypeRef; use dpp::data_contract::DataContractFactory; use dpp::document::serialization_traits::DocumentPlatformConversionMethodsV0; use dpp::document::{Document, DocumentV0, DocumentV0Getters, DocumentV0Setters}; +use dpp::fee::fee_result::FeeResult; use dpp::platform_value::{platform_value, Identifier, Value}; use dpp::prelude::DataContract; use dpp::version::PlatformVersion; +use std::borrow::Cow; use std::collections::BTreeMap; /// One hour in each of the two units these tests deal in: `*_SECONDS` @@ -1839,3 +1841,1865 @@ fn ranked_chain_below_bucket_update_materializes_like_insert() { ); } } + +/// The TTL lifecycle end to end — `book/src/drive/time-range-ttl.md` +/// exercised through the real walkers: +/// +/// * a bucket-creating write drops buckets behind the horizon (and only +/// those: a bucket starting exactly AT the horizon survives); +/// * the per-write cap amortizes catch-up instead of dumping a backlog +/// on one writer; +/// * deleting and updating a document whose buckets were dropped +/// succeeds — the removal side skips exactly the dropped buckets, and +/// an update never resurrects one; +/// * ranked per-window leaderboards below the bucket ride along: live +/// windows keep serving, dropped windows take their secondaries with +/// them (the recursive-delete placeholder sweeps indexed axes). +#[test] +fn ttl_drops_expired_buckets_and_walkers_skip_them() { + use crate::drive::document::paths::contract_document_type_path_vec; + use crate::fees::op::LowLevelDriveOperation; + use crate::util::grove_operations::DirectQueryType; + use dpp::data_contract::document_type::DocumentPropertyType; + use grovedb_path::SubtreePath; + + let platform_version = PlatformVersion::latest(); + let drive = setup_drive_with_initial_state_structure(Some(platform_version)); + + // Tumbling 2h windows, TTL 4h, ranked hashtags per window. + let factory = + DataContractFactory::new(PlatformVersion::latest().protocol_version).expect("factory"); + let index_map = vec![ + ( + Value::Text("name".to_string()), + Value::Text("trendingTtl".to_string()), + ), + ( + Value::Text("properties".to_string()), + Value::Array(vec![ + platform_value!({"$createdAt": "asc"}), + platform_value!({"hashtag": "asc"}), + ]), + ), + ( + Value::Text("timeRange".to_string()), + Value::Map(vec![ + ( + Value::Text("on".to_string()), + Value::Text("$createdAt".to_string()), + ), + ( + Value::Text("range".to_string()), + Value::U64(2 * HOUR_SECONDS), + ), + ( + Value::Text("step".to_string()), + Value::U64(2 * HOUR_SECONDS), + ), + (Value::Text("ttl".to_string()), Value::U64(4 * HOUR_SECONDS)), + ]), + ), + ( + Value::Text("countable".to_string()), + Value::Text("countable".to_string()), + ), + (Value::Text("rangeCountable".to_string()), Value::Bool(true)), + ( + Value::Text("rankedCountable".to_string()), + Value::Bool(true), + ), + ]; + let document_schema = platform_value!({ + "type": "object", + "properties": { + "hashtag": {"type": "string", "maxLength": 61, "position": 0}, + }, + "required": ["hashtag", "$createdAt"], + "indices": Value::Array(vec![Value::Map(index_map)]), + "additionalProperties": false, + }); + let schemas = platform_value!({ "post": document_schema }); + let contract = factory + .create_with_value_config(Identifier::from([203u8; 32]), 0, schemas, None, None) + .expect("a TTL'd ranked windowed index registers") + .data_contract_owned(); + + drive + .apply_contract( + &contract, + BlockInfo::default(), + true, + StorageFlags::optional_default_as_cow(), + None, + platform_version, + ) + .expect("apply contract"); + + let document_type = contract.document_type_for_name("post").expect("post"); + let transform = document_type + .indexes() + .get("trendingTtl") + .expect("trendingTtl index") + .time_range + .clone() + .expect("transform"); + assert_eq!(transform.ttl_seconds, Some(4 * HOUR_SECONDS)); + + let mut level_path = contract_document_type_path_vec(contract.id_ref().as_bytes(), "post"); + level_path.push(transform.storage_key("$createdAt").into_bytes()); + + let bucket_exists = |start_ms: u64| -> bool { + let path_refs: Vec<&[u8]> = level_path + .iter() + .map(|segment| segment.as_slice()) + .collect(); + let key = DocumentPropertyType::encode_date_timestamp(start_ms); + let mut ops: Vec = vec![]; + drive + .grove_has_raw( + SubtreePath::from(path_refs.as_slice()), + key.as_slice(), + DirectQueryType::StatefulDirectQuery, + None, + &mut ops, + &platform_version.drive, + ) + .expect("existence check") + }; + + let insert_at = |created_at: u64, tag: &str| -> Document { + let owner_bytes = fixture_bytes(5, created_at, tag); + let document = Document::V0(DocumentV0 { + id: Identifier::from(fixture_bytes(6, created_at, tag)), + owner_id: Identifier::from(owner_bytes), + properties: BTreeMap::from([("hashtag".to_string(), Value::Text(tag.to_string()))]), + created_at: Some(created_at), + revision: Some(1), + ..Default::default() + }); + drive + .add_document_for_contract( + DocumentAndContractInfo { + owned_document_info: OwnedDocumentInfo { + document_info: DocumentRefInfo(( + &document, + StorageFlags::optional_default_as_cow(), + )), + owner_id: Some(owner_bytes), + }, + contract: &contract, + document_type, + }, + false, + BlockInfo { + time_ms: created_at, + ..Default::default() + }, + true, + None, + platform_version, + None, + ) + .expect("add document"); + document + }; + + let h = HOUR_MS; + // Anchor away from the epoch so horizons never underflow. + let t0 = 1_000 * h; + + let doc_a = insert_at(t0 + 10 * MINUTE_MS_TTL, "alpha"); // bucket t0 + let mut doc_bravo = insert_at(t0 + 2 * h + 10 * MINUTE_MS_TTL, "bravo"); // bucket t0+2h + assert!(bucket_exists(t0), "nothing is expired yet"); + + // Writing at exactly t0+6h (bucket t0+6h) puts the horizon at + // exactly t0+2h: bucket t0 (start < horizon) is dropped, bucket + // t0+2h (start == horizon) survives — expiry is strictly-below. + insert_at(t0 + 6 * h, "charlie"); + assert!( + !bucket_exists(t0), + "the bucket behind the horizon must be dropped by the bucket-creating write" + ); + assert!( + bucket_exists(t0 + 2 * h), + "a bucket starting exactly at the horizon is not expired" + ); + assert!(bucket_exists(t0 + 6 * h)); + + // Deleting a document whose buckets were dropped must succeed: the + // removal side skips exactly the dropped buckets. + drive + .delete_document_for_contract( + doc_a.id(), + &contract, + "post", + BlockInfo { + time_ms: t0 + 6 * h + 20 * MINUTE_MS_TTL, + ..Default::default() + }, + true, + None, + platform_version, + None, + ) + .expect("deleting a document whose windows were dropped succeeds"); + + // Updating a document whose windows have all expired succeeds and + // never resurrects a dropped bucket. `bravo`'s bucket (t0+2h) is + // still standing here; move time far enough that it has been dropped + // first, then update it. + insert_at(t0 + 10 * h + 10 * MINUTE_MS_TTL, "delta"); // horizon now t0+6h + assert!( + !bucket_exists(t0 + 2 * h), + "catch-up cleanup drops the next expired bucket" + ); + + // Updating the document whose bucket was dropped exercises the update + // walker's expired-window paths for real: the new entry keys filter to + // nothing (its windows are all expired), the old-entry loop skips the + // dropped bucket, and — the invariant that protects the flat-drop + // path-reuse contract — the dropped bucket is NOT resurrected. + doc_bravo.set("hashtag", Value::Text("bravo2".to_string())); + doc_bravo.set_revision(Some(2)); + drive + .update_document_for_contract( + &doc_bravo, + &contract, + document_type, + None, + BlockInfo { + time_ms: t0 + 10 * h + 15 * MINUTE_MS_TTL, + ..Default::default() + }, + true, + None, + None, + platform_version, + None, + ) + .expect("updating a document whose windows were all dropped succeeds"); + assert!( + !bucket_exists(t0 + 2 * h), + "the update must not resurrect the dropped bucket" + ); + + let mut doc_b = insert_at(t0 + 10 * h + 20 * MINUTE_MS_TTL, "echo"); + // Update a LIVE document normally (control), then delete it — the + // full mutable lifecycle stays intact under a TTL'd index. + doc_b.set("hashtag", Value::Text("echo2".to_string())); + doc_b.set_revision(Some(2)); + drive + .update_document_for_contract( + &doc_b, + &contract, + document_type, + None, + BlockInfo { + time_ms: t0 + 10 * h + 30 * MINUTE_MS_TTL, + ..Default::default() + }, + true, + None, + None, + platform_version, + None, + ) + .expect("updating a live document under a TTL'd index succeeds"); + + // The live window's per-window leaderboard serves after all of the + // above: ranked entries for bucket t0+10h are (delta 1, echo2 1) — + // and echo (the pre-update suffix) is gone. + { + use crate::query::drive_document_ranked_query::index_picker::resolve_ranked_query_for_mode; + use crate::query::drive_document_ranked_query::PrefixPin; + use crate::query::{DocumentRankedMode, RankedAxis}; + let mode = DocumentRankedMode { + axis: RankedAxis::Count, + descending: true, + k: 10, + offset: 0, + group_by_property: "hashtag".to_string(), + aggregate_field: String::new(), + prefix_pins: vec![PrefixPin { + field: "$createdAt".to_string(), + values: vec![Value::U64(t0 + 10 * h)], + }], + }; + let ranked_query = resolve_ranked_query_for_mode( + contract.id().to_buffer(), + document_type, + "post".to_string(), + document_type.indexes(), + &mode, + &created_at_resolution(document_type), + platform_version, + ) + .expect("the TTL'd ranked index covers the pinned request"); + let page = ranked_query + .execute_top_k_no_proof(&drive, None, platform_version) + .expect("the live window's leaderboard reads"); + let keys: Vec<&[u8]> = page.entries.iter().map(|e| e.key.as_slice()).collect(); + assert!(keys.contains(&b"delta".as_slice())); + assert!(keys.contains(&b"echo2".as_slice())); + assert!(!keys.contains(&b"echo".as_slice())); + } +} + +/// One minute in milliseconds, for the TTL lifecycle test's offsets. +const MINUTE_MS_TTL: u64 = 60_000; + +/// The TTL grammar rejections that need contract-level context: the +/// SystemLimits cap, and two indexes sharing a grid with different TTLs +/// (one storage level cannot have two lifecycles). The structural lower +/// bound (`ttl >= range`) is covered at the `Index` parse level. +#[test] +fn ttl_contract_level_rejections() { + let factory = + DataContractFactory::new(PlatformVersion::latest().protocol_version).expect("factory"); + let time_range_with_ttl = |ttl: u64| { + Value::Map(vec![ + ( + Value::Text("on".to_string()), + Value::Text("$createdAt".to_string()), + ), + (Value::Text("range".to_string()), Value::U64(HOUR_SECONDS)), + (Value::Text("step".to_string()), Value::U64(HOUR_SECONDS)), + (Value::Text("ttl".to_string()), Value::U64(ttl)), + ]) + }; + let schema_with_indices = |indices: Value| { + platform_value!({ + "post": { + "type": "object", + "properties": { + "hashtag": {"type": "string", "maxLength": 61, "position": 0}, + }, + "required": ["hashtag", "$createdAt"], + "indices": indices, + "additionalProperties": false, + } + }) + }; + + // Over the one-week cap. + let over_cap = schema_with_indices(Value::Array(vec![Value::Map(vec![ + ( + Value::Text("name".to_string()), + Value::Text("overCap".to_string()), + ), + ( + Value::Text("properties".to_string()), + Value::Array(vec![ + platform_value!({"$createdAt": "asc"}), + platform_value!({"hashtag": "asc"}), + ]), + ), + ( + Value::Text("timeRange".to_string()), + time_range_with_ttl(604_800 + 1), + ), + ( + Value::Text("countable".to_string()), + Value::Text("countable".to_string()), + ), + ])])); + let error = factory + .create_with_value_config(Identifier::from([204u8; 32]), 0, over_cap, None, None) + .expect_err("a TTL over the cap must be refused"); + assert!( + error.to_string().contains("exceeds the maximum"), + "expected the cap rejection, got: {error}" + ); + + // Same grid, different TTLs. + let index = |name: &str, ttl: u64| { + Value::Map(vec![ + ( + Value::Text("name".to_string()), + Value::Text(name.to_string()), + ), + ( + Value::Text("properties".to_string()), + Value::Array(vec![ + platform_value!({"$createdAt": "asc"}), + platform_value!({"hashtag": "asc"}), + ]), + ), + ( + Value::Text("timeRange".to_string()), + time_range_with_ttl(ttl), + ), + ( + Value::Text("countable".to_string()), + Value::Text("countable".to_string()), + ), + ]) + }; + let conflicting = schema_with_indices(Value::Array(vec![ + index("gridA", 3 * HOUR_SECONDS), + index("gridB", 4 * HOUR_SECONDS), + ])); + let error = factory + .create_with_value_config(Identifier::from([205u8; 32]), 0, conflicting, None, None) + .expect_err("one grid cannot carry two lifecycles"); + assert!( + error.to_string().contains("two lifecycles"), + "expected the shared-grid TTL conflict rejection, got: {error}" + ); +} + +/// Budgeted drainage across writes: a bucket whose drop-operation count +/// exceeds one write's budget stands PARTIALLY drained until later writes +/// finish it — groups leave deepest-first in key order — and document +/// removal keeps working through every intermediate state, at full-path +/// granularity: a doc whose group the drain already took deletes as a +/// clean skip, one whose group still stands deletes normally. +#[test] +fn ttl_partial_drain_resumes_across_writes_and_removals_stay_exact() { + use crate::drive::document::paths::contract_document_type_path_vec; + use crate::fees::op::LowLevelDriveOperation; + use crate::util::grove_operations::DirectQueryType; + use dpp::data_contract::document_type::DocumentPropertyType; + use grovedb_path::SubtreePath; + + let platform_version = PlatformVersion::latest(); + let drive = setup_drive_with_initial_state_structure(Some(platform_version)); + + // Same shape as the lifecycle test: tumbling 2h windows, TTL 4h, + // ranked hashtags per window. + let factory = + DataContractFactory::new(PlatformVersion::latest().protocol_version).expect("factory"); + let index_map = vec![ + ( + Value::Text("name".to_string()), + Value::Text("trendingTtl".to_string()), + ), + ( + Value::Text("properties".to_string()), + Value::Array(vec![ + platform_value!({"$createdAt": "asc"}), + platform_value!({"hashtag": "asc"}), + ]), + ), + ( + Value::Text("timeRange".to_string()), + Value::Map(vec![ + ( + Value::Text("on".to_string()), + Value::Text("$createdAt".to_string()), + ), + ( + Value::Text("range".to_string()), + Value::U64(2 * HOUR_SECONDS), + ), + ( + Value::Text("step".to_string()), + Value::U64(2 * HOUR_SECONDS), + ), + (Value::Text("ttl".to_string()), Value::U64(4 * HOUR_SECONDS)), + ]), + ), + ( + Value::Text("countable".to_string()), + Value::Text("countable".to_string()), + ), + (Value::Text("rangeCountable".to_string()), Value::Bool(true)), + ( + Value::Text("rankedCountable".to_string()), + Value::Bool(true), + ), + ]; + let document_schema = platform_value!({ + "type": "object", + "properties": { + "hashtag": {"type": "string", "maxLength": 61, "position": 0}, + }, + "required": ["hashtag", "$createdAt"], + "indices": Value::Array(vec![Value::Map(index_map)]), + "additionalProperties": false, + }); + let schemas = platform_value!({ "post": document_schema }); + let contract = factory + .create_with_value_config(Identifier::from([206u8; 32]), 0, schemas, None, None) + .expect("contract registers") + .data_contract_owned(); + drive + .apply_contract( + &contract, + BlockInfo::default(), + true, + StorageFlags::optional_default_as_cow(), + None, + platform_version, + ) + .expect("apply contract"); + + let document_type = contract.document_type_for_name("post").expect("post"); + let transform = document_type + .indexes() + .get("trendingTtl") + .expect("index") + .time_range + .clone() + .expect("transform"); + + let mut level_path = contract_document_type_path_vec(contract.id_ref().as_bytes(), "post"); + level_path.push(transform.storage_key("$createdAt").into_bytes()); + + let path_exists = |segments: &[Vec]| -> bool { + let (key, parents) = segments.split_last().expect("non-empty path"); + let parent_refs: Vec<&[u8]> = parents.iter().map(|segment| segment.as_slice()).collect(); + let mut ops: Vec = vec![]; + drive + .grove_has_raw( + SubtreePath::from(parent_refs.as_slice()), + key.as_slice(), + DirectQueryType::StatefulDirectQuery, + None, + &mut ops, + &platform_version.drive, + ) + .expect("existence check") + }; + + let insert_at = |created_at: u64, tag: &str| -> Document { + let owner_bytes = fixture_bytes(7, created_at, tag); + let document = Document::V0(DocumentV0 { + id: Identifier::from(fixture_bytes(8, created_at, tag)), + owner_id: Identifier::from(owner_bytes), + properties: BTreeMap::from([("hashtag".to_string(), Value::Text(tag.to_string()))]), + created_at: Some(created_at), + revision: Some(1), + ..Default::default() + }); + drive + .add_document_for_contract( + DocumentAndContractInfo { + owned_document_info: OwnedDocumentInfo { + document_info: DocumentRefInfo(( + &document, + StorageFlags::optional_default_as_cow(), + )), + owner_id: Some(owner_bytes), + }, + contract: &contract, + document_type, + }, + false, + BlockInfo { + time_ms: created_at, + ..Default::default() + }, + true, + None, + platform_version, + None, + ) + .expect("add document"); + document + }; + + let h = HOUR_MS; + let t0 = 2_000 * h; + let old_bucket_key = DocumentPropertyType::encode_date_timestamp(t0); + + // Enough groups that full drainage exceeds one write's budget: each + // group costs 2 drop operations ([0] drop + value-tree delete), plus + // the property-name drop and the bucket drop — budget/2 + 1 groups + // puts the total at budget + 4. + let budget = platform_version + .system_limits + .max_time_range_ttl_drop_operations_per_write + .expect("PV14 declares a drain budget") as u64; + let groups = budget / 2 + 1; + let docs: Vec = (1..=groups) + .map(|i| insert_at(t0 + i * MINUTE_MS_TTL, &format!("g{i:02}"))) + .collect(); + + // First write past the horizon: the budget drains the first budget/2 + // groups (2 ops each) and stops — the bucket stands, partially + // drained, with the last group and the property-name tree intact. + insert_at(t0 + 6 * h, "w1"); + let bucket_path = { + let mut path = level_path.clone(); + path.push(old_bucket_key.clone()); + path + }; + assert!( + path_exists(&bucket_path), + "the bucket stands after the budget ran out" + ); + let group_path = |tag: &str| -> Vec> { + let mut path = bucket_path.clone(); + path.push(b"hashtag".to_vec()); + path.push(tag.as_bytes().to_vec()); + path + }; + for i in 1..=(budget / 2) { + let gone = format!("g{i:02}"); + assert!( + !path_exists(&group_path(&gone)), + "group {gone} drains in the first write" + ); + } + let last_group = format!("g{groups:02}"); + assert!( + path_exists(&group_path(&last_group)), + "the budget stops before {last_group}" + ); + + // A document whose group the drain took deletes as a clean skip; one + // whose group still stands deletes normally. Both under the standing, + // partially drained bucket. + for (doc, label) in [ + (&docs[0], "drained group"), + (docs.last().expect("groups is nonzero"), "standing group"), + ] { + drive + .delete_document_for_contract( + doc.id(), + &contract, + "post", + BlockInfo { + time_ms: t0 + 6 * h + 10 * MINUTE_MS_TTL, + ..Default::default() + }, + true, + None, + platform_version, + None, + ) + .unwrap_or_else(|e| panic!("deleting a doc from a {label} must succeed: {e:?}")); + } + + // The next write finishes whatever drainage the deletes' own up-tree + // pruning left behind; the bucket is gone. + insert_at(t0 + 6 * h + 20 * MINUTE_MS_TTL, "w2"); + assert!( + !path_exists(&bucket_path), + "drainage completes across writes" + ); +} + +/// Shared builder for the TTL parent-layout matrix: a 2h/2h grid with +/// `ttl: 4h` over `[$createdAt, hashtag]`, with the aggregate keywords +/// supplied per case, plus an integer `amount` property for the sum- +/// bearing layouts. +fn build_ttl_contract_with_index_keys( + seed: u8, + extra_index_keys: Vec<(Value, Value)>, +) -> DataContract { + build_time_range_contract_with_index_keys(seed, Some(4 * HOUR_SECONDS), extra_index_keys) +} + +/// Same contract shape with the TTL declaration as the only degree of +/// freedom, so a TTL'd index and its standing twin are byte-for-byte +/// comparable in fee tests. +fn build_time_range_contract_with_index_keys( + seed: u8, + ttl_seconds: Option, + extra_index_keys: Vec<(Value, Value)>, +) -> DataContract { + let factory = + DataContractFactory::new(PlatformVersion::latest().protocol_version).expect("factory"); + let mut index_map = vec![ + ( + Value::Text("name".to_string()), + Value::Text("trendingTtl".to_string()), + ), + ( + Value::Text("properties".to_string()), + Value::Array(vec![ + platform_value!({"$createdAt": "asc"}), + platform_value!({"hashtag": "asc"}), + ]), + ), + ( + Value::Text("timeRange".to_string()), + Value::Map(vec![ + ( + Value::Text("on".to_string()), + Value::Text("$createdAt".to_string()), + ), + ( + Value::Text("range".to_string()), + Value::U64(2 * HOUR_SECONDS), + ), + ( + Value::Text("step".to_string()), + Value::U64(2 * HOUR_SECONDS), + ), + ]), + ), + ]; + if let Some(ttl) = ttl_seconds { + let Some((_, Value::Map(time_range_map))) = index_map.last_mut() else { + panic!("timeRange map is the last base index key"); + }; + time_range_map.push((Value::Text("ttl".to_string()), Value::U64(ttl))); + } + index_map.extend(extra_index_keys); + let document_schema = platform_value!({ + "type": "object", + "properties": { + // 59: the Avg axis's 16-byte sort key tightens the ranked + // group-key cap below the Count axis's 61. + "hashtag": {"type": "string", "maxLength": 59, "position": 0}, + "amount": {"type": "integer", "minimum": 0, "maximum": 4294967295u64, "position": 1}, + }, + "required": ["hashtag", "amount", "$createdAt"], + "indices": Value::Array(vec![Value::Map(index_map)]), + "additionalProperties": false, + }); + let schemas = platform_value!({ "post": document_schema }); + factory + .create_with_value_config(Identifier::from([seed; 32]), 0, schemas, None, None) + .expect("contract registers") + .data_contract_owned() +} + +/// One full TTL drain cycle against a contract: two groups in a doomed +/// bucket, one write past the horizon, and the bucket must be gone — +/// exercising whichever node-removal arm the index's aggregate keywords +/// select. Returns after asserting absence. +fn run_ttl_drain_cycle(contract: &DataContract) { + use crate::drive::document::paths::contract_document_type_path_vec; + use crate::fees::op::LowLevelDriveOperation; + use crate::util::grove_operations::DirectQueryType; + use dpp::data_contract::document_type::DocumentPropertyType; + use grovedb_path::SubtreePath; + + let platform_version = PlatformVersion::latest(); + let drive = setup_drive_with_initial_state_structure(Some(platform_version)); + drive + .apply_contract( + contract, + BlockInfo::default(), + true, + StorageFlags::optional_default_as_cow(), + None, + platform_version, + ) + .expect("apply contract"); + let document_type = contract.document_type_for_name("post").expect("post"); + let transform = document_type + .indexes() + .get("trendingTtl") + .expect("index") + .time_range + .clone() + .expect("transform"); + + let insert_at = |created_at: u64, tag: &str| { + let owner_bytes = fixture_bytes(9, created_at, tag); + let document = Document::V0(DocumentV0 { + id: Identifier::from(fixture_bytes(10, created_at, tag)), + owner_id: Identifier::from(owner_bytes), + properties: BTreeMap::from([ + ("hashtag".to_string(), Value::Text(tag.to_string())), + ("amount".to_string(), Value::U64(5)), + ]), + created_at: Some(created_at), + revision: Some(1), + ..Default::default() + }); + drive + .add_document_for_contract( + DocumentAndContractInfo { + owned_document_info: OwnedDocumentInfo { + document_info: DocumentRefInfo(( + &document, + StorageFlags::optional_default_as_cow(), + )), + owner_id: Some(owner_bytes), + }, + contract, + document_type, + }, + false, + BlockInfo { + time_ms: created_at, + ..Default::default() + }, + true, + None, + platform_version, + None, + ) + .expect("add document"); + }; + + let h = HOUR_MS; + let t0 = 3_000 * h; + insert_at(t0 + MINUTE_MS_TTL, "aa"); + insert_at(t0 + 2 * MINUTE_MS_TTL, "bb"); + insert_at(t0 + 6 * h, "live"); + + let mut level_path = contract_document_type_path_vec(contract.id_ref().as_bytes(), "post"); + level_path.push(transform.storage_key("$createdAt").into_bytes()); + let path_refs: Vec<&[u8]> = level_path + .iter() + .map(|segment| segment.as_slice()) + .collect(); + let mut ops: Vec = vec![]; + let doomed = drive + .grove_has_raw( + SubtreePath::from(path_refs.as_slice()), + DocumentPropertyType::encode_date_timestamp(t0).as_slice(), + DirectQueryType::StatefulDirectQuery, + None, + &mut ops, + &platform_version.drive, + ) + .expect("existence check"); + assert!( + !doomed, + "the doomed bucket drains within one write's budget for this layout" + ); +} + +/// Comment-19 matrix: drainage must be exercised for every node-removal +/// arm — the flat-drop fallback under a plain (non-ranked) property-name +/// tree, and the three dedicated indexed-tree deletes. +#[test] +fn ttl_drainage_covers_every_parent_layout() { + // Plain parent: countable only — value trees leave via the flat drop. + run_ttl_drain_cycle(&build_ttl_contract_with_index_keys( + 210, + vec![( + Value::Text("countable".to_string()), + Value::Text("countable".to_string()), + )], + )); + // ProvableCountIndexedTree parent (rankedCountable). + run_ttl_drain_cycle(&build_ttl_contract_with_index_keys( + 211, + vec![ + ( + Value::Text("countable".to_string()), + Value::Text("countable".to_string()), + ), + (Value::Text("rangeCountable".to_string()), Value::Bool(true)), + ( + Value::Text("rankedCountable".to_string()), + Value::Bool(true), + ), + ], + )); + // ProvableSumIndexedTree parent (rankedSummable over `amount`). + run_ttl_drain_cycle(&build_ttl_contract_with_index_keys( + 212, + vec![ + ( + Value::Text("summable".to_string()), + Value::Text("amount".to_string()), + ), + (Value::Text("rangeSummable".to_string()), Value::Bool(true)), + (Value::Text("rankedSummable".to_string()), Value::Bool(true)), + ], + )); + // ProvableCountProvableSumIndexedTree parent (rankedAverageable). + run_ttl_drain_cycle(&build_ttl_contract_with_index_keys( + 213, + vec![ + ( + Value::Text("averageable".to_string()), + Value::Text("amount".to_string()), + ), + ( + Value::Text("rangeAverageable".to_string()), + Value::Bool(true), + ), + ( + Value::Text("rankedAverageable".to_string()), + Value::Bool(true), + ), + ], + )); +} + +/// Comment-17 regression: a drainage budget that runs out immediately +/// after dropping a group's flat `[0]` tree leaves the group's value tree +/// standing without it. Deleting that group's document must then skip at +/// the `[0]` granularity — every shallower segment of its entry path +/// still exists — and a later drain finishes the bucket. +#[test] +fn ttl_budget_boundary_after_zero_tree_keeps_deletes_exact() { + use crate::drive::document::paths::contract_document_type_path_vec; + use crate::fees::op::LowLevelDriveOperation; + use crate::util::grove_operations::DirectQueryType; + use dpp::data_contract::document_type::DocumentPropertyType; + use grovedb_path::SubtreePath; + + let platform_version = PlatformVersion::latest(); + let drive = setup_drive_with_initial_state_structure(Some(platform_version)); + let contract = build_ttl_contract_with_index_keys( + 214, + vec![ + ( + Value::Text("countable".to_string()), + Value::Text("countable".to_string()), + ), + (Value::Text("rangeCountable".to_string()), Value::Bool(true)), + ( + Value::Text("rankedCountable".to_string()), + Value::Bool(true), + ), + ], + ); + drive + .apply_contract( + &contract, + BlockInfo::default(), + true, + StorageFlags::optional_default_as_cow(), + None, + platform_version, + ) + .expect("apply contract"); + let document_type = contract.document_type_for_name("post").expect("post"); + let transform = document_type + .indexes() + .get("trendingTtl") + .expect("index") + .time_range + .clone() + .expect("transform"); + let storage_key = transform.storage_key("$createdAt"); + let bucket_level = document_type + .index_structure() + .sub_levels() + .get(&storage_key) + .expect("the grid level exists in the index structure"); + + let h = HOUR_MS; + let t0 = 4_000 * h; + let owner_bytes = fixture_bytes(11, t0, "solo"); + let document = Document::V0(DocumentV0 { + id: Identifier::from(fixture_bytes(12, t0, "solo")), + owner_id: Identifier::from(owner_bytes), + properties: BTreeMap::from([ + ("hashtag".to_string(), Value::Text("solo".to_string())), + ("amount".to_string(), Value::U64(5)), + ]), + created_at: Some(t0 + MINUTE_MS_TTL), + revision: Some(1), + ..Default::default() + }); + drive + .add_document_for_contract( + DocumentAndContractInfo { + owned_document_info: OwnedDocumentInfo { + document_info: DocumentRefInfo(( + &document, + StorageFlags::optional_default_as_cow(), + )), + owner_id: Some(owner_bytes), + }, + contract: &contract, + document_type, + }, + false, + BlockInfo { + time_ms: t0 + MINUTE_MS_TTL, + ..Default::default() + }, + true, + None, + platform_version, + None, + ) + .expect("add document"); + + let mut level_path = contract_document_type_path_vec(contract.id_ref().as_bytes(), "post"); + level_path.push(storage_key.clone().into_bytes()); + let after_expiry_ms = t0 + 6 * h; + + // Budget 1: exactly the group's `[0]` tree drops; its value tree + // stands without it. + drive + .drain_expired_time_range_buckets( + &transform, + bucket_level, + &level_path, + after_expiry_ms, + 1, + None, + platform_version, + ) + .expect("a budget of one drops exactly the [0] tree"); + let exists = |segments: &[Vec]| -> bool { + let (key, parents) = segments.split_last().expect("non-empty"); + let parent_refs: Vec<&[u8]> = parents.iter().map(|segment| segment.as_slice()).collect(); + let mut ops: Vec = vec![]; + drive + .grove_has_raw( + SubtreePath::from(parent_refs.as_slice()), + key.as_slice(), + DirectQueryType::StatefulDirectQuery, + None, + &mut ops, + &platform_version.drive, + ) + .expect("existence check") + }; + let bucket_key = DocumentPropertyType::encode_date_timestamp(t0); + let mut group_path = level_path.clone(); + group_path.push(bucket_key.clone()); + group_path.push(b"hashtag".to_vec()); + group_path.push(b"solo".to_vec()); + let mut zero_path = group_path.clone(); + zero_path.push(vec![0]); + assert!(exists(&group_path), "the value tree stands"); + assert!(!exists(&zero_path), "its [0] tree is gone"); + + // The delete must skip at [0] granularity rather than target the + // missing subtree. + drive + .delete_document_for_contract( + document.id(), + &contract, + "post", + BlockInfo { + time_ms: after_expiry_ms, + ..Default::default() + }, + true, + None, + platform_version, + None, + ) + .expect("deleting a doc whose [0] tree drained must succeed"); + + // A later drain finishes the bucket. + drive + .drain_expired_time_range_buckets( + &transform, + bucket_level, + &level_path, + after_expiry_ms, + 16, + None, + platform_version, + ) + .expect("the rest of the bucket drains"); + let mut bucket_path = level_path.clone(); + bucket_path.push(bucket_key); + assert!(!exists(&bucket_path), "the bucket is gone"); +} + +/// Comment-14 (blocking) regression: drainage is unbilled system +/// maintenance, so a write that performs it must never cost more than +/// its estimate — the `estimated >= actual` invariant that validation's +/// balance check depends on. The estimate runs first (it does not read +/// or mutate state), then the same write applies while draining an +/// expired bucket. +#[test] +fn ttl_draining_write_never_exceeds_its_estimate() { + let platform_version = PlatformVersion::latest(); + let drive = setup_drive_with_initial_state_structure(Some(platform_version)); + let contract = build_ttl_contract_with_index_keys( + 215, + vec![ + ( + Value::Text("countable".to_string()), + Value::Text("countable".to_string()), + ), + (Value::Text("rangeCountable".to_string()), Value::Bool(true)), + ( + Value::Text("rankedCountable".to_string()), + Value::Bool(true), + ), + ], + ); + drive + .apply_contract( + &contract, + BlockInfo::default(), + true, + StorageFlags::optional_default_as_cow(), + None, + platform_version, + ) + .expect("apply contract"); + let document_type = contract.document_type_for_name("post").expect("post"); + + let h = HOUR_MS; + let t0 = 5_000 * h; + let make_doc = |created_at: u64, tag: &str| -> Document { + Document::V0(DocumentV0 { + id: Identifier::from(fixture_bytes(14, created_at, tag)), + owner_id: Identifier::from(fixture_bytes(13, created_at, tag)), + properties: BTreeMap::from([ + ("hashtag".to_string(), Value::Text(tag.to_string())), + ("amount".to_string(), Value::U64(5)), + ]), + created_at: Some(created_at), + revision: Some(1), + ..Default::default() + }) + }; + let add = |document: &Document, apply: bool| -> dpp::fee::fee_result::FeeResult { + let owner_bytes = document.owner_id().to_buffer(); + drive + .add_document_for_contract( + DocumentAndContractInfo { + owned_document_info: OwnedDocumentInfo { + document_info: DocumentRefInfo(( + document, + StorageFlags::optional_default_as_cow(), + )), + owner_id: Some(owner_bytes), + }, + contract: &contract, + document_type, + }, + false, + BlockInfo { + time_ms: document.created_at().expect("created at"), + ..Default::default() + }, + apply, + None, + platform_version, + None, + ) + .expect("add document") + }; + + // Seed the doomed bucket, then a write far enough ahead that applying + // it drains that bucket. + add(&make_doc(t0 + MINUTE_MS_TTL, "old"), true); + let draining_doc = make_doc(t0 + 6 * h, "fresh"); + let estimated = add(&draining_doc, false); + let actual = add(&draining_doc, true); + assert!( + estimated.storage_fee >= actual.storage_fee, + "storage: estimated {} must cover actual {}", + estimated.storage_fee, + actual.storage_fee + ); + assert!( + estimated.processing_fee >= actual.processing_fee, + "processing: estimated {} must cover actual {} — drainage must not \ + bill the triggering write beyond its estimate", + estimated.processing_fee, + actual.processing_fee + ); +} + +/// Drainage rides updates too, not only inserts: with no insert ever +/// touching the index again, a lone update past the horizon must drop +/// the expired bucket — including the one the updated document's own +/// entries lived in, whose old-entry removal then skips coherently. +#[test] +fn ttl_update_only_write_drains_expired_buckets() { + use crate::drive::document::paths::contract_document_type_path_vec; + use crate::fees::op::LowLevelDriveOperation; + use crate::util::grove_operations::DirectQueryType; + use dpp::data_contract::document_type::DocumentPropertyType; + use grovedb_path::SubtreePath; + + let platform_version = PlatformVersion::latest(); + let drive = setup_drive_with_initial_state_structure(Some(platform_version)); + let contract = build_ttl_contract_with_index_keys( + 216, + vec![ + ( + Value::Text("countable".to_string()), + Value::Text("countable".to_string()), + ), + (Value::Text("rangeCountable".to_string()), Value::Bool(true)), + ( + Value::Text("rankedCountable".to_string()), + Value::Bool(true), + ), + ], + ); + drive + .apply_contract( + &contract, + BlockInfo::default(), + true, + StorageFlags::optional_default_as_cow(), + None, + platform_version, + ) + .expect("apply contract"); + let document_type = contract.document_type_for_name("post").expect("post"); + let transform = document_type + .indexes() + .get("trendingTtl") + .expect("index") + .time_range + .clone() + .expect("transform"); + + let h = HOUR_MS; + let t0 = 6_000 * h; + let owner_bytes = fixture_bytes(15, t0, "only"); + let mut document = Document::V0(DocumentV0 { + id: Identifier::from(fixture_bytes(16, t0, "only")), + owner_id: Identifier::from(owner_bytes), + properties: BTreeMap::from([ + ("hashtag".to_string(), Value::Text("only".to_string())), + ("amount".to_string(), Value::U64(5)), + ]), + created_at: Some(t0 + MINUTE_MS_TTL), + revision: Some(1), + ..Default::default() + }); + drive + .add_document_for_contract( + DocumentAndContractInfo { + owned_document_info: OwnedDocumentInfo { + document_info: DocumentRefInfo(( + &document, + StorageFlags::optional_default_as_cow(), + )), + owner_id: Some(owner_bytes), + }, + contract: &contract, + document_type, + }, + false, + BlockInfo { + time_ms: t0 + MINUTE_MS_TTL, + ..Default::default() + }, + true, + None, + platform_version, + None, + ) + .expect("add document"); + + // The only write after expiry is an UPDATE. + document.set("hashtag", Value::Text("only2".to_string())); + document.set_revision(Some(2)); + drive + .update_document_for_contract( + &document, + &contract, + document_type, + Some(owner_bytes), + BlockInfo { + time_ms: t0 + 6 * h, + ..Default::default() + }, + true, + None, + None, + platform_version, + None, + ) + .expect("an update past the horizon succeeds and drains"); + + let mut level_path = contract_document_type_path_vec(contract.id_ref().as_bytes(), "post"); + level_path.push(transform.storage_key("$createdAt").into_bytes()); + let path_refs: Vec<&[u8]> = level_path + .iter() + .map(|segment| segment.as_slice()) + .collect(); + let mut ops: Vec = vec![]; + let bucket_stands = drive + .grove_has_raw( + SubtreePath::from(path_refs.as_slice()), + DocumentPropertyType::encode_date_timestamp(t0).as_slice(), + DirectQueryType::StatefulDirectQuery, + None, + &mut ops, + &platform_version.drive, + ) + .expect("existence check"); + assert!( + !bucket_stands, + "an update-only write must drain the expired bucket" + ); +} + +/// The ephemeral-bytes fee reclassification, measured against a standing +/// twin: two contracts identical byte-for-byte except that one declares a +/// `ttl`. The TTL'd insert's index bytes must leave the storage fee (only +/// the primary document row still bills there) and land in processing at +/// the ephemeral-bytes rate; deleting the document must refund strictly +/// less, because flagless ephemeral index bytes have nothing to refund. +/// Estimation stays an upper bound in both classes through the split +/// batch. +#[test] +fn ttl_index_bytes_bill_to_processing_without_refunds() { + let platform_version = PlatformVersion::latest(); + let drive = setup_drive_with_initial_state_structure(Some(platform_version)); + let index_keys = || { + vec![ + ( + Value::Text("countable".to_string()), + Value::Text("countable".to_string()), + ), + (Value::Text("rangeCountable".to_string()), Value::Bool(true)), + ( + Value::Text("rankedCountable".to_string()), + Value::Bool(true), + ), + ] + }; + let ttl_contract = + build_time_range_contract_with_index_keys(217, Some(4 * HOUR_SECONDS), index_keys()); + let standing_contract = build_time_range_contract_with_index_keys(218, None, index_keys()); + // Same document schema with no indexes at all: its insert pays for the + // primary document row alone, giving the exact storage fee a TTL'd + // contract must match if its index bytes truly bill zero storage. + let index_free_contract = { + let factory = + DataContractFactory::new(PlatformVersion::latest().protocol_version).expect("factory"); + let document_schema = platform_value!({ + "type": "object", + "properties": { + "hashtag": {"type": "string", "maxLength": 59, "position": 0}, + "amount": {"type": "integer", "minimum": 0, "maximum": 4294967295u64, "position": 1}, + }, + "required": ["hashtag", "amount", "$createdAt"], + "additionalProperties": false, + }); + factory + .create_with_value_config( + Identifier::from([219u8; 32]), + 0, + platform_value!({ "post": document_schema }), + None, + None, + ) + .expect("contract registers") + .data_contract_owned() + }; + for contract in [&ttl_contract, &standing_contract, &index_free_contract] { + drive + .apply_contract( + contract, + BlockInfo::default(), + true, + StorageFlags::optional_default_as_cow(), + None, + platform_version, + ) + .expect("apply contract"); + } + + let t0 = 5_000 * HOUR_MS; + let block_info = BlockInfo { + time_ms: t0, + ..Default::default() + }; + let make_doc = || -> Document { + Document::V0(DocumentV0 { + id: Identifier::from(fixture_bytes(17, t0, "twin")), + owner_id: Identifier::from(fixture_bytes(18, t0, "twin")), + properties: BTreeMap::from([ + ("hashtag".to_string(), Value::Text("twin".to_string())), + ("amount".to_string(), Value::U64(5)), + ]), + created_at: Some(t0), + revision: Some(1), + ..Default::default() + }) + }; + let add = |contract: &DataContract, apply: bool| -> FeeResult { + let document = make_doc(); + let owner_bytes = document.owner_id().to_buffer(); + // Owner-carrying flags, so standing index bytes produce visible + // refunds on delete — the contrast the TTL side must not show. + let storage_flags = Cow::Owned(StorageFlags::SingleEpochOwned(0, owner_bytes)); + drive + .add_document_for_contract( + DocumentAndContractInfo { + owned_document_info: OwnedDocumentInfo { + document_info: DocumentRefInfo((&document, Some(storage_flags))), + owner_id: Some(owner_bytes), + }, + contract, + document_type: contract.document_type_for_name("post").expect("post"), + }, + false, + block_info, + apply, + None, + platform_version, + None, + ) + .expect("add document") + }; + + let ttl_estimated = add(&ttl_contract, false); + let ttl_insert = add(&ttl_contract, true); + let standing_insert = add(&standing_contract, true); + let index_free_insert = add(&index_free_contract, true); + + assert!( + ttl_insert.storage_fee < standing_insert.storage_fee, + "TTL'd index bytes must leave the storage fee: {} vs standing {}", + ttl_insert.storage_fee, + standing_insert.storage_fee + ); + assert!( + ttl_insert.storage_fee > 0, + "the primary document row still bills to storage" + ); + assert!( + ttl_insert.processing_fee > standing_insert.processing_fee, + "the ephemeral-bytes rate must land in processing: {} vs standing {}", + ttl_insert.processing_fee, + standing_insert.processing_fee + ); + assert_eq!( + ttl_insert.storage_fee, index_free_insert.storage_fee, + "with a TTL, index writes must contribute exactly zero storage: the \ + storage fee must equal an index-free contract's" + ); + assert!( + ttl_estimated.storage_fee >= ttl_insert.storage_fee + && ttl_estimated.processing_fee >= ttl_insert.processing_fee, + "estimation must stay an upper bound in both fee classes through the \ + split batch: estimated ({}, {}) vs actual ({}, {})", + ttl_estimated.storage_fee, + ttl_estimated.processing_fee, + ttl_insert.storage_fee, + ttl_insert.processing_fee + ); + + let doc_id = make_doc().id(); + let delete = |contract: &DataContract| -> FeeResult { + drive + .delete_document_for_contract( + doc_id, + contract, + "post", + block_info, + true, + None, + platform_version, + None, + ) + .expect("delete document") + }; + let ttl_delete = delete(&ttl_contract); + let standing_delete = delete(&standing_contract); + let index_free_delete = delete(&index_free_contract); + let refund_total = |fee_result: &FeeResult| -> u64 { + fee_result + .fee_refunds + .clone() + .sum_per_epoch() + .into_values() + .sum() + }; + assert!( + refund_total(&ttl_delete) < refund_total(&standing_delete), + "flagless ephemeral index bytes must not refund: ttl {} vs standing {}", + refund_total(&ttl_delete), + refund_total(&standing_delete) + ); + assert_eq!( + refund_total(&ttl_delete), + refund_total(&index_free_delete), + "a TTL'd delete refunds exactly the primary document row — the same \ + as a contract with no indexes at all" + ); +} + +/// Several indexes may share one grid-qualified level (same grid, same +/// ttl); the walkers must drain that level exactly ONCE per write, before +/// any batch mutation is queued. The per-index regression: four countable +/// indexes share `$createdAt#7200#7200` and the bucket holds enough groups +/// that a full drain exceeds one budget — a per-index drain would keep +/// dropping paths (directly) that an earlier index's queued removals +/// target, failing batch apply with `InvalidPath`, while spending up to +/// four budgets. One update past the horizon must succeed against the +/// partially drained bucket, and later writes finish the job. +#[test] +fn ttl_shared_grid_drains_once_per_write() { + use crate::drive::document::paths::contract_document_type_path_vec; + use crate::fees::op::LowLevelDriveOperation; + use crate::util::grove_operations::DirectQueryType; + use dpp::data_contract::document_type::DocumentPropertyType; + use grovedb_path::SubtreePath; + + let platform_version = PlatformVersion::latest(); + let drive = setup_drive_with_initial_state_structure(Some(platform_version)); + + let factory = + DataContractFactory::new(PlatformVersion::latest().protocol_version).expect("factory"); + let shared_grid_index = |name: &str, second_property: &str| -> Value { + Value::Map(vec![ + ( + Value::Text("name".to_string()), + Value::Text(name.to_string()), + ), + ( + Value::Text("properties".to_string()), + Value::Array(vec![ + platform_value!({"$createdAt": "asc"}), + platform_value!({second_property: "asc"}), + ]), + ), + ( + Value::Text("timeRange".to_string()), + Value::Map(vec![ + ( + Value::Text("on".to_string()), + Value::Text("$createdAt".to_string()), + ), + ( + Value::Text("range".to_string()), + Value::U64(2 * HOUR_SECONDS), + ), + ( + Value::Text("step".to_string()), + Value::U64(2 * HOUR_SECONDS), + ), + (Value::Text("ttl".to_string()), Value::U64(4 * HOUR_SECONDS)), + ]), + ), + ( + Value::Text("countable".to_string()), + Value::Text("countable".to_string()), + ), + ]) + }; + let document_schema = platform_value!({ + "type": "object", + "properties": { + "hashtag": {"type": "string", "maxLength": 59, "position": 0}, + "amount": {"type": "integer", "minimum": 0, "maximum": 4294967295u64, "position": 1}, + "alpha": {"type": "string", "maxLength": 59, "position": 2}, + "beta": {"type": "string", "maxLength": 59, "position": 3}, + }, + "required": ["hashtag", "amount", "alpha", "beta", "$createdAt"], + // Index names order the walker's per-index loop (BTreeMap), while + // drainage walks property-name trees in KEY order — so `aHashtag` + // iterates FIRST while its `hashtag` tree drains LAST. Under a + // per-index drain that is the poison ordering: the first index + // queues removals against the still-standing hashtag entries, then + // a later index's drain drops them directly and batch apply fails + // with InvalidPath. + "indices": Value::Array(vec![ + shared_grid_index("aHashtag", "hashtag"), + shared_grid_index("bAmount", "amount"), + shared_grid_index("cAlpha", "alpha"), + shared_grid_index("dBeta", "beta"), + ]), + "additionalProperties": false, + }); + let contract = factory + .create_with_value_config( + Identifier::from([220u8; 32]), + 0, + platform_value!({ "post": document_schema }), + None, + None, + ) + .expect("four indexes sharing one grid and ttl validate") + .data_contract_owned(); + drive + .apply_contract( + &contract, + BlockInfo::default(), + true, + StorageFlags::optional_default_as_cow(), + None, + platform_version, + ) + .expect("apply contract"); + let document_type = contract.document_type_for_name("post").expect("post"); + let transform = document_type + .indexes() + .get("aHashtag") + .expect("index") + .time_range + .clone() + .expect("transform"); + + let h = HOUR_MS; + let t0 = 7_000 * h; + let owner_bytes = fixture_bytes(19, t0, "zeta"); + let mut document = Document::V0(DocumentV0 { + id: Identifier::from(fixture_bytes(20, t0, "zeta")), + owner_id: Identifier::from(owner_bytes), + properties: BTreeMap::from([ + ("hashtag".to_string(), Value::Text("zeta".to_string())), + ("amount".to_string(), Value::U64(4)), + ("alpha".to_string(), Value::Text("four".to_string())), + ("beta".to_string(), Value::Text("nine".to_string())), + ]), + created_at: Some(t0 + MINUTE_MS_TTL), + revision: Some(1), + ..Default::default() + }); + drive + .add_document_for_contract( + DocumentAndContractInfo { + owned_document_info: OwnedDocumentInfo { + document_info: DocumentRefInfo(( + &document, + StorageFlags::optional_default_as_cow(), + )), + owner_id: Some(owner_bytes), + }, + contract: &contract, + document_type, + }, + false, + BlockInfo { + time_ms: t0 + MINUTE_MS_TTL, + ..Default::default() + }, + true, + None, + platform_version, + None, + ) + .expect("add document"); + + // Seed enough further groups that draining the bucket exceeds one + // write's budget: with G distinct values per property, the bucket + // costs 4 property-name trees x 2G + 4 + 1 = 8G + 5 drop operations, + // so G = budget/8 + 1 guarantees a partial first drain. + let budget = platform_version + .system_limits + .max_time_range_ttl_drop_operations_per_write + .expect("PV14 declares a drain budget") as u64; + let extra_groups = budget / 8; + for i in 1..=extra_groups { + let extra_owner = fixture_bytes(24, t0 + i, "extra"); + let extra = Document::V0(DocumentV0 { + id: Identifier::from(fixture_bytes(25, t0 + i, "extra")), + owner_id: Identifier::from(extra_owner), + properties: BTreeMap::from([ + ("hashtag".to_string(), Value::Text(format!("tag{i:02}"))), + ("amount".to_string(), Value::U64(100 + i)), + ("alpha".to_string(), Value::Text(format!("alp{i:02}"))), + ("beta".to_string(), Value::Text(format!("bet{i:02}"))), + ]), + created_at: Some(t0 + MINUTE_MS_TTL + i), + revision: Some(1), + ..Default::default() + }); + drive + .add_document_for_contract( + DocumentAndContractInfo { + owned_document_info: OwnedDocumentInfo { + document_info: DocumentRefInfo(( + &extra, + StorageFlags::optional_default_as_cow(), + )), + owner_id: Some(extra_owner), + }, + contract: &contract, + document_type, + }, + false, + BlockInfo { + time_ms: t0 + MINUTE_MS_TTL + i, + ..Default::default() + }, + true, + None, + platform_version, + None, + ) + .expect("add extra group document"); + } + let total_drop_operations = 8 * (extra_groups + 1) + 5; + + // First write past the horizon: one budget cannot finish the bucket — + // a partial drain is guaranteed, and every index's queued removals + // must stay consistent with it. + let mut update_at = |time_ms: u64, revision: u64, hashtag: &str| { + document.set("hashtag", Value::Text(hashtag.to_string())); + document.set_revision(Some(revision)); + drive + .update_document_for_contract( + &document, + &contract, + document_type, + Some(owner_bytes), + BlockInfo { + time_ms, + ..Default::default() + }, + true, + None, + None, + platform_version, + None, + ) + .expect("an update against a partially drained shared-grid bucket succeeds"); + }; + let mut level_path = contract_document_type_path_vec(contract.id_ref().as_bytes(), "post"); + level_path.push(transform.storage_key("$createdAt").into_bytes()); + let bucket_stands = || -> bool { + let path_refs: Vec<&[u8]> = level_path + .iter() + .map(|segment| segment.as_slice()) + .collect(); + let mut ops: Vec = vec![]; + drive + .grove_has_raw( + SubtreePath::from(path_refs.as_slice()), + DocumentPropertyType::encode_date_timestamp(t0).as_slice(), + DirectQueryType::StatefulDirectQuery, + None, + &mut ops, + &platform_version.drive, + ) + .expect("existence check") + }; + + update_at(t0 + 6 * h, 2, "zeta2"); + // One budget cannot finish the bucket, so it must still stand here — a + // per-index drain (four budgets in one write) would already have + // removed it. + assert!( + bucket_stands(), + "one write spends exactly one budget, so the bucket survives the \ + first update" + ); + let finishing_writes = total_drop_operations.div_ceil(budget); + for write in 1..finishing_writes { + update_at( + t0 + 6 * h + write * MINUTE_MS_TTL, + 2 + write, + &format!("zeta{write}"), + ); + } + assert!( + !bucket_stands(), + "{finishing_writes} writes' budgets must finish the \ + {total_drop_operations}-op shared bucket" + ); +} + +/// The every-write cleanup rule includes DELETE-only writes: an index +/// receiving nothing but deletions must still advance drainage. Two +/// documents share one expired bucket (6 drop operations — within one +/// 8-op budget); deleting one past the horizon must take the whole +/// bucket, other document's expired entries included. +#[test] +fn ttl_delete_only_write_drains_expired_buckets() { + use crate::drive::document::paths::contract_document_type_path_vec; + use crate::fees::op::LowLevelDriveOperation; + use crate::util::grove_operations::DirectQueryType; + use dpp::data_contract::document_type::DocumentPropertyType; + use grovedb_path::SubtreePath; + + let platform_version = PlatformVersion::latest(); + let drive = setup_drive_with_initial_state_structure(Some(platform_version)); + let contract = build_ttl_contract_with_index_keys( + 221, + vec![( + Value::Text("countable".to_string()), + Value::Text("countable".to_string()), + )], + ); + drive + .apply_contract( + &contract, + BlockInfo::default(), + true, + StorageFlags::optional_default_as_cow(), + None, + platform_version, + ) + .expect("apply contract"); + let document_type = contract.document_type_for_name("post").expect("post"); + let transform = document_type + .indexes() + .get("trendingTtl") + .expect("index") + .time_range + .clone() + .expect("transform"); + + let h = HOUR_MS; + let t0 = 8_000 * h; + let insert = |tag: &str, seed: u8| -> Identifier { + let owner_bytes = fixture_bytes(seed, t0, tag); + let document = Document::V0(DocumentV0 { + id: Identifier::from(fixture_bytes(seed.wrapping_add(1), t0, tag)), + owner_id: Identifier::from(owner_bytes), + properties: BTreeMap::from([ + ("hashtag".to_string(), Value::Text(tag.to_string())), + ("amount".to_string(), Value::U64(5)), + ]), + created_at: Some(t0 + MINUTE_MS_TTL), + revision: Some(1), + ..Default::default() + }); + drive + .add_document_for_contract( + DocumentAndContractInfo { + owned_document_info: OwnedDocumentInfo { + document_info: DocumentRefInfo(( + &document, + StorageFlags::optional_default_as_cow(), + )), + owner_id: Some(owner_bytes), + }, + contract: &contract, + document_type, + }, + false, + BlockInfo { + time_ms: t0 + MINUTE_MS_TTL, + ..Default::default() + }, + true, + None, + platform_version, + None, + ) + .expect("add document"); + document.id() + }; + let doomed_id = insert("doomed", 21); + insert("survivor", 23); + + // The only write after expiry is a DELETE. + drive + .delete_document_for_contract( + doomed_id, + &contract, + "post", + BlockInfo { + time_ms: t0 + 6 * h, + ..Default::default() + }, + true, + None, + platform_version, + None, + ) + .expect("a delete past the horizon succeeds and drains"); + + let mut level_path = contract_document_type_path_vec(contract.id_ref().as_bytes(), "post"); + level_path.push(transform.storage_key("$createdAt").into_bytes()); + let path_refs: Vec<&[u8]> = level_path + .iter() + .map(|segment| segment.as_slice()) + .collect(); + let mut ops: Vec = vec![]; + let bucket_stands = drive + .grove_has_raw( + SubtreePath::from(path_refs.as_slice()), + DocumentPropertyType::encode_date_timestamp(t0).as_slice(), + DirectQueryType::StatefulDirectQuery, + None, + &mut ops, + &platform_version.drive, + ) + .expect("existence check"); + assert!( + !bucket_stands, + "a delete-only write must drain the expired bucket, the surviving \ + document's entries included" + ); +} diff --git a/packages/rs-drive/src/drive/document/insert/add_document_for_contract_operations/v0/mod.rs b/packages/rs-drive/src/drive/document/insert/add_document_for_contract_operations/v0/mod.rs index 4144f65a705..36f4e24361a 100644 --- a/packages/rs-drive/src/drive/document/insert/add_document_for_contract_operations/v0/mod.rs +++ b/packages/rs-drive/src/drive/document/insert/add_document_for_contract_operations/v0/mod.rs @@ -54,6 +54,7 @@ impl Drive { &document_and_contract_info, previous_batch_operations, estimated_costs_only_with_layer_info, + block_info.time_ms, transaction, &mut batch_operations, platform_version, @@ -151,6 +152,7 @@ impl Drive { &document_and_contract_info, previous_batch_operations, estimated_costs_only_with_layer_info, + block_info.time_ms, transaction, &mut batch_operations, platform_version, diff --git a/packages/rs-drive/src/drive/document/insert/add_document_for_contract_operations/v1/mod.rs b/packages/rs-drive/src/drive/document/insert/add_document_for_contract_operations/v1/mod.rs index e659e61d72a..48c408e1aca 100644 --- a/packages/rs-drive/src/drive/document/insert/add_document_for_contract_operations/v1/mod.rs +++ b/packages/rs-drive/src/drive/document/insert/add_document_for_contract_operations/v1/mod.rs @@ -61,6 +61,7 @@ impl Drive { &document_and_contract_info, previous_batch_operations, estimated_costs_only_with_layer_info, + block_info.time_ms, transaction, &mut batch_operations, platform_version, @@ -167,6 +168,7 @@ impl Drive { &document_and_contract_info, previous_batch_operations, estimated_costs_only_with_layer_info, + block_info.time_ms, transaction, &mut batch_operations, platform_version, diff --git a/packages/rs-drive/src/drive/document/insert/add_indices_for_top_index_level_for_contract_operations/mod.rs b/packages/rs-drive/src/drive/document/insert/add_indices_for_top_index_level_for_contract_operations/mod.rs index b065bccdb9c..c5c7cd54534 100644 --- a/packages/rs-drive/src/drive/document/insert/add_indices_for_top_index_level_for_contract_operations/mod.rs +++ b/packages/rs-drive/src/drive/document/insert/add_indices_for_top_index_level_for_contract_operations/mod.rs @@ -39,6 +39,7 @@ impl Drive { estimated_costs_only_with_layer_info: &mut Option< HashMap, >, + block_time_ms: u64, transaction: TransactionArg, batch_operations: &mut Vec, platform_version: &PlatformVersion, @@ -70,6 +71,7 @@ impl Drive { document_and_contract_info, previous_batch_operations, estimated_costs_only_with_layer_info, + block_time_ms, transaction, batch_operations, platform_version, diff --git a/packages/rs-drive/src/drive/document/insert/add_indices_for_top_index_level_for_contract_operations/v2/mod.rs b/packages/rs-drive/src/drive/document/insert/add_indices_for_top_index_level_for_contract_operations/v2/mod.rs index 3b23787770c..a3e1291ce87 100644 --- a/packages/rs-drive/src/drive/document/insert/add_indices_for_top_index_level_for_contract_operations/v2/mod.rs +++ b/packages/rs-drive/src/drive/document/insert/add_indices_for_top_index_level_for_contract_operations/v2/mod.rs @@ -45,6 +45,7 @@ impl Drive { /// v14 shared-prefix aggregate fix — see /// [`Drive::add_indices_for_index_level_for_contract_operations_v2`] /// for the full story. + #[allow(clippy::too_many_arguments)] pub(super) fn add_indices_for_top_index_level_for_contract_operations_v2( &self, document_and_contract_info: &DocumentAndContractInfo, @@ -52,6 +53,7 @@ impl Drive { estimated_costs_only_with_layer_info: &mut Option< HashMap, >, + block_time_ms: u64, transaction: TransactionArg, batch_operations: &mut Vec, platform_version: &PlatformVersion, @@ -121,6 +123,22 @@ impl Drive { let property_name_tree_type = tree_types.property_name_tree_type; let value_tree_type = tree_types.value_tree_type; + // TTL'd sub-levels write EPHEMERAL state: their operations are + // collected locally and re-tagged so they apply in their own + // batch and bill their bytes to processing at the ephemeral + // rate instead of to storage, and their elements carry NO + // storage flags — no refunds ever accrue against bytes the + // drain later drops unmetered. Everything else is unchanged. + let sub_level_is_ephemeral = sub_level + .time_range() + .is_some_and(|transform| transform.ttl_seconds.is_some()); + let mut ephemeral_local_operations: Vec = vec![]; + let index_storage_flags = if sub_level_is_ephemeral { + None + } else { + storage_flags + }; + // at this point the contract path is to the contract documents // for each index the top index component will already have been added // when the contract itself was created @@ -219,7 +237,7 @@ impl Drive { estimated_layer_sizes: AllSubtrees( document_top_field_estimated_size as u8, estimated_sum_trees_for_value_tree_type(value_tree_type), - storage_flags.map(|s| s.serialized_size()), + index_storage_flags.map(|s| s.serialized_size()), ), }, ); @@ -249,18 +267,51 @@ impl Drive { .unwrap_or(1), ); + // TTL drainage rides every write into a TTL'd index: a bounded + // number of deepest-first drop operations against the oldest + // expired bucket, resuming wherever the previous write's budget + // ran out. When nothing is expired this is one bounded range + // read. Stateful only — the estimation dry run neither reads + // state nor prices drops (each is O(1); the count is capped). + if estimated_costs_only_with_layer_info.is_none() { + if let Some(transform) = sub_level.time_range() { + if transform.ttl_seconds.is_some() { + if let Some(max_operations) = platform_version + .system_limits + .max_time_range_ttl_drop_operations_per_write + { + self.drain_expired_time_range_buckets( + transform, + sub_level, + &index_path, + block_time_ms, + max_operations, + transaction, + platform_version, + )?; + } + } + } + } + let bucket_count = index_keys.len(); for (bucket, index_key) in index_keys.into_iter().enumerate() { // The zero will not matter here, because the PathKeyInfo is variable let path_key_info = index_key.clone().add_path::<0>(index_path.clone()); + let index_batch_operations: &mut Vec = + if sub_level_is_ephemeral { + &mut ephemeral_local_operations + } else { + &mut *batch_operations + }; self.batch_insert_empty_tree_if_not_exists( path_key_info, value_tree_type, - storage_flags, + index_storage_flags, value_apply_type, transaction, previous_batch_operations, - batch_operations, + index_batch_operations, drive_version, )?; @@ -291,6 +342,12 @@ impl Drive { // just inserted forward as the recursive level's // `parent_value_tree_type` so its continuation children pick // the right zero-contribution op. + let index_batch_operations: &mut Vec = + if sub_level_is_ephemeral { + &mut ephemeral_local_operations + } else { + &mut *batch_operations + }; self.add_indices_for_index_level_for_contract_operations( document_and_contract_info, index_path_info, @@ -299,14 +356,22 @@ impl Drive { all_fields_null, value_tree_type, previous_batch_operations, - &storage_flags, + &index_storage_flags, estimated_costs_only_with_layer_info, event_id, transaction, - batch_operations, + index_batch_operations, platform_version, )?; } + + if sub_level_is_ephemeral { + batch_operations.extend( + ephemeral_local_operations + .into_iter() + .map(LowLevelDriveOperation::retag_ephemeral), + ); + } } Ok(()) } diff --git a/packages/rs-drive/src/drive/document/mod.rs b/packages/rs-drive/src/drive/document/mod.rs index c553c5a797e..0c520675855 100644 --- a/packages/rs-drive/src/drive/document/mod.rs +++ b/packages/rs-drive/src/drive/document/mod.rs @@ -61,6 +61,11 @@ pub(crate) mod ranked_index_tree_type; #[cfg(feature = "server")] pub(crate) mod index_level_tree_types; +/// Shared TTL semantics for time-range indexes — see +/// `book/src/drive/time-range-ttl.md`. +#[cfg(feature = "server")] +pub(crate) mod time_range_ttl; + /// indexOnly entry probes: entry path/key derivation shared by the write /// path and the ABCI state-validation probes #[cfg(feature = "server")] diff --git a/packages/rs-drive/src/drive/document/time_range_ttl.rs b/packages/rs-drive/src/drive/document/time_range_ttl.rs new file mode 100644 index 00000000000..097b3c39f05 --- /dev/null +++ b/packages/rs-drive/src/drive/document/time_range_ttl.rs @@ -0,0 +1,525 @@ +//! Shared TTL semantics for time-range indexes — the walker-facing half +//! of `book/src/drive/time-range-ttl.md`. +//! +//! Three rules, one definition each: +//! +//! - **Writes never target expired buckets.** The insert path cannot +//! produce one by construction (`$createdAt` &co. are consensus-assigned +//! and validation requires `ttl >= range`), and the update path filters +//! its new entry keys through [`live_time_range_entry_keys`] — an update +//! of a document whose windows have all expired simply leaves it with no +//! entries under the TTL'd index, and never resurrects a dropped bucket. +//! - **Removals touch an expired bucket only while it still stands.** The +//! TTL drop is bucket-granular and lazy, so between a bucket's expiry +//! and its drop a delete (or key-changing update) of one of its +//! documents must still remove that document's entries — otherwise the +//! bucket would carry dangling references until the drop. Once the +//! bucket is gone the entries are gone with it, and per-entry removal +//! must skip rather than fail. [`Drive::time_range_entry_is_removable`] +//! is that check: live bucket ⇒ always removable; expired bucket ⇒ +//! removable exactly when it still exists. The existence read is +//! deterministic — it reads consensus state. +//! - **Expiry has one definition**: +//! [`TimeRangeTransform::expiry_horizon_ms`], shared by these helpers +//! and the bucket-drop cleanup. +//! +//! # Billing +//! +//! Nothing in this module bills the triggering user: drainage operations +//! and the walkers' TTL bookkeeping reads accumulate their costs into +//! local scratch vectors that are dropped, never into the caller's fee +//! operations. This is load-bearing for the `estimated >= actual` fee +//! invariant — the estimation dry run cannot read state, so it cannot +//! price state-dependent drainage, and billing it on execution only would +//! let a transition pass validation and then overdraw on apply. The work +//! itself is bounded (a capped count of O(1) operations plus a handful of +//! bounded reads per write) and is system maintenance of state nobody +//! holds refunds against; the planned ephemeral-bytes fee rate is where +//! TTL writers pre-pay it in aggregate. + +use crate::drive::document::index_level_tree_types::index_level_tree_types_with_continuation_demotion; +use crate::drive::Drive; +use crate::error::Error; +use crate::fees::op::LowLevelDriveOperation; +use crate::util::grove_operations::push_drive_operation_result; +use crate::util::grove_operations::DirectQueryType; +use dpp::data_contract::document_type::{DocumentPropertyType, IndexLevel, TimeRangeTransform}; +use dpp::version::PlatformVersion; +use grovedb::query_result_type::QueryResultType; +use grovedb::{PathQuery, Query, SizedQuery, TransactionArg, TreeType}; +use grovedb_path::SubtreePath; + +/// The bucket start a stored time-range entry key encodes, when it +/// encodes one. Mirrors the gate in +/// [`TimeRangeTransform::entry_keys_for_raw`]: only an exactly-8-byte key +/// is a bucket start — the null entry (empty key) and raw non-timestamp +/// keys have no expiry semantics and live until their document goes. +pub(crate) fn entry_key_bucket_start(entry_key: &[u8]) -> Option { + (entry_key.len() == 8) + .then(|| DocumentPropertyType::decode_date_timestamp(entry_key)) + .flatten() +} + +/// Filter a derived time-range entry-key set down to the keys whose +/// bucket has not expired at `block_time_ms`. Keys without bucket-start +/// semantics (the null entry, raw keys) always pass; everything passes +/// when the transform declares no TTL. +pub(crate) fn live_time_range_entry_keys( + transform: &TimeRangeTransform, + entry_keys: Vec>, + block_time_ms: u64, +) -> Vec> { + let Some(horizon) = transform.expiry_horizon_ms(block_time_ms) else { + return entry_keys; + }; + entry_keys + .into_iter() + .filter(|key| entry_key_bucket_start(key).is_none_or(|start| start >= horizon)) + .collect() +} + +impl Drive { + /// Whether a removal walker should process the time-range entry at + /// `entry_key` under the grid level at `level_path`. + /// + /// `true` for every live (or non-bucket) key with no read performed; + /// for an expired key, `true` exactly when the bucket value tree still + /// exists — the window between expiry and its lazy drop, where the + /// document's entries are still on disk and must still be removed. + /// Callers in estimation mode must not call this (state reads have no + /// place in a dry run); they process every key, which keeps the dry + /// run an upper bound. + pub(crate) fn time_range_entry_is_removable( + &self, + transform: &TimeRangeTransform, + entry_key: &[u8], + block_time_ms: u64, + level_path: &[Vec], + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result { + // Unbilled bookkeeping read — see the module's Billing section. + let mut scratch_operations: Vec = vec![]; + let Some(horizon) = transform.expiry_horizon_ms(block_time_ms) else { + return Ok(true); + }; + let Some(start) = entry_key_bucket_start(entry_key) else { + return Ok(true); + }; + if start >= horizon { + return Ok(true); + } + let path_refs: Vec<&[u8]> = level_path + .iter() + .map(|segment| segment.as_slice()) + .collect(); + self.grove_has_raw( + SubtreePath::from(path_refs.as_slice()), + entry_key, + DirectQueryType::StatefulDirectQuery, + transaction, + &mut scratch_operations, + &platform_version.drive, + ) + } + + /// Whether every segment of `path_segments` beyond the first + /// `known_prefix_len` (a prefix known to exist — the contract's + /// document-type path) resolves, walked one `has_raw` at a time so a + /// missing intermediate subtree answers `false` instead of erroring. + /// + /// The removal walkers use this at full-path granularity for entries + /// in expired-but-standing buckets: TTL drainage removes whole `[0]` + /// trees and group value trees before the bucket itself goes, so an + /// entry's deeper path can be gone while the bucket still stands. + pub(crate) fn expired_entry_path_exists( + &self, + path_segments: &[Vec], + known_prefix_len: usize, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result { + // Unbilled bookkeeping reads — see the module's Billing section. + let mut scratch_operations: Vec = vec![]; + let mut depth = known_prefix_len; + while depth < path_segments.len() { + let parent_refs: Vec<&[u8]> = path_segments[..depth] + .iter() + .map(|segment| segment.as_slice()) + .collect(); + if !self.grove_has_raw( + SubtreePath::from(parent_refs.as_slice()), + path_segments[depth].as_slice(), + DirectQueryType::StatefulDirectQuery, + transaction, + &mut scratch_operations, + &platform_version.drive, + )? { + return Ok(false); + } + depth += 1; + } + Ok(true) + } + + /// Drain expired buckets from the grid level at `level_path` — the + /// lazy, budgeted cleanup every write into a TTL'd index continues. + /// + /// The drop primitive is grovedb's flat-subtree drop (grovedb#848 / + /// PR #849): O(1) consensus removal of a subtree **declared to hold no + /// child subtrees**, with its storage prefixes reclaimed outside + /// consensus via range tombstones. A time-range bucket is NOT flat — + /// it nests one property-name tree per remaining index level, value + /// trees per distinct group, and `[0]` reference trees — so the drain + /// works **deepest-first**, exactly one flat unit at a time: + /// + /// - a `[0]` reference tree holds only reference/Item elements → flat + /// drop (this is where the mass lives); + /// - a value tree whose property-name children and `[0]` subtree are + /// gone holds at most bare elements → flat drop — except under a + /// ranked (indexed-primary) property-name tree, where grovedb + /// rightly refuses a generic child removal and the (by then empty) + /// tree leaves through the ordinary delete, which mirrors the + /// secondary; + /// - a drained property-name tree → flat drop, which also dooms its + /// per-axis secondary prefixes when it was an indexed primary; + /// - the emptied bucket itself → flat drop. + /// + /// Every step is a deterministic function of consensus state and block + /// time, and every step is O(1) — the *number* of steps is what scales + /// with user data (one per group, per level, per bucket), and that is + /// exactly what `max_operations` bounds per write. A bucket drains + /// across as many writes as it needs; between writes it stands + /// partially drained, which TTL semantics allow (entries live *at + /// most* `ttl`) and which the removal walkers handle at full-path + /// granularity. + /// + /// The dropped paths embed their window start, so they are never + /// re-created before their redo records drain (writes never target + /// expired buckets) — the flat-drop path-reuse contract holds by + /// construction. The host completes reclamation by calling + /// `GroveDb::flush_pending_prefix_drops` after committing the block's + /// transaction (and once at startup). + /// One budgeted drainage pass for every TTL'd time-range level of a + /// document type. Levels are keyed by their grid-qualified storage key, + /// so indexes sharing a grid share one level and drain exactly once per + /// write — draining per *index* would multiply the per-write budget and, + /// worse, interleave direct grovedb drops with already-queued batch + /// mutations (a later index's drain can remove a path an earlier + /// index's pending operation targets). Callers must therefore run this + /// sweep BEFORE queuing any batch mutations, so every queued operation + /// describes post-drain state. Stateful only — never call from an + /// estimation dry run. + pub(crate) fn drain_expired_time_range_levels( + &self, + index_level: &IndexLevel, + contract_document_type_path: &[Vec], + block_time_ms: u64, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result<(), Error> { + let Some(max_operations) = platform_version + .system_limits + .max_time_range_ttl_drop_operations_per_write + else { + return Ok(()); + }; + for (name, sub_level) in index_level.sub_levels() { + if let Some(transform) = sub_level.time_range() { + if transform.ttl_seconds.is_some() { + let mut level_path = contract_document_type_path.to_vec(); + level_path.push(name.as_bytes().to_vec()); + self.drain_expired_time_range_buckets( + transform, + sub_level, + &level_path, + block_time_ms, + max_operations, + transaction, + platform_version, + )?; + } + } + } + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + pub(crate) fn drain_expired_time_range_buckets( + &self, + transform: &TimeRangeTransform, + bucket_level: &IndexLevel, + level_path: &[Vec], + block_time_ms: u64, + max_operations: u16, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result<(), Error> { + let Some(horizon) = transform.expiry_horizon_ms(block_time_ms) else { + return Ok(()); + }; + // Unbilled system maintenance — see the module's Billing section. + let drive_operations: &mut Vec = &mut vec![]; + let mut budget = max_operations; + while budget > 0 { + // Oldest expired bucket first. The null entry (empty key) + // sorts below every bucket start and is excluded — null + // entries are not windowed and live until their document goes. + let horizon_key = DocumentPropertyType::encode_date_timestamp(horizon); + let mut below_horizon = Query::new(); + // Only 8-byte keys carry bucket semantics. With today's grammar + // no other key can sort below the horizon anyway — the source + // is a required system timestamp, so the level holds 8-byte + // bucket starts plus at most the single null entry (empty key) + // — but the range start and the wider limit keep the finder + // live even if a future grammar admits raw (non-timestamp) + // keys: without them, low-sorting raw keys could fill every + // result slot and stall drainage forever. + below_horizon.insert_range(vec![0u8; 8]..horizon_key); + let path_query = PathQuery::new( + level_path.to_vec(), + SizedQuery::new(below_horizon, Some(8), None), + ); + let (results, _) = self.grove_get_raw_path_query( + &path_query, + transaction, + QueryResultType::QueryKeyElementPairResultType, + drive_operations, + &platform_version.drive, + )?; + let Some(bucket_key) = results + .to_key_elements() + .into_iter() + .map(|(key, _)| key) + .find(|key| entry_key_bucket_start(key).is_some_and(|start| start < horizon)) + else { + return Ok(()); + }; + let mut bucket_path = level_path.to_vec(); + bucket_path.push(bucket_key.clone()); + let fully_drained = self.drain_expired_node( + &bucket_path, + bucket_level, + level_path, + &bucket_key, + // The grid level is never an indexed primary — ranking the + // bucketed level is rejected at contract validation. + TreeType::NormalTree, + &mut budget, + transaction, + drive_operations, + platform_version, + )?; + if !fully_drained { + return Ok(()); + } + } + Ok(()) + } + + /// Drain one tree of an expired bucket, deepest-first, then drop the + /// tree itself. Returns whether the tree was fully removed (`false` ⇒ + /// the budget ran out mid-way; the next write resumes exactly here, + /// because every completed step is a real removal). + /// + /// `level` describes the merged contract-known structure below this + /// tree (property-name children by level key); the value-tree children + /// under each property-name tree are user data, enumerated one at a + /// time. + #[allow(clippy::too_many_arguments)] + fn drain_expired_node( + &self, + node_path: &[Vec], + level: &IndexLevel, + parent_path: &[Vec], + node_key: &[u8], + parent_tree_type: TreeType, + budget: &mut u16, + transaction: TransactionArg, + drive_operations: &mut Vec, + platform_version: &PlatformVersion, + ) -> Result { + let drive_version = &platform_version.drive; + // 1) Contract-known property-name children. + for (level_key, sub_level) in level.sub_levels() { + let level_key_bytes = level_key.as_bytes(); + let node_path_refs: Vec<&[u8]> = + node_path.iter().map(|segment| segment.as_slice()).collect(); + let pn_element = self.grove_get_raw_optional( + SubtreePath::from(node_path_refs.as_slice()), + level_key_bytes, + DirectQueryType::StatefulDirectQuery, + transaction, + drive_operations, + drive_version, + )?; + if pn_element.is_none() { + continue; + } + let pn_tree_type = index_level_tree_types_with_continuation_demotion(sub_level)? + .property_name_tree_type; + let mut pn_path = node_path.to_vec(); + pn_path.push(level_key_bytes.to_vec()); + // 1a) User-data value-tree children, one at a time. + loop { + if *budget == 0 { + return Ok(false); + } + let mut all = Query::new(); + all.insert_all(); + let path_query = + PathQuery::new(pn_path.clone(), SizedQuery::new(all, Some(1), None)); + let (results, _) = self.grove_get_raw_path_query( + &path_query, + transaction, + QueryResultType::QueryKeyElementPairResultType, + drive_operations, + drive_version, + )?; + let Some((value_key, _)) = results.to_key_elements().into_iter().next() else { + break; + }; + let mut value_path = pn_path.clone(); + value_path.push(value_key.clone()); + if !self.drain_expired_node( + &value_path, + sub_level, + &pn_path, + &value_key, + pn_tree_type, + budget, + transaction, + drive_operations, + platform_version, + )? { + return Ok(false); + } + } + // 1b) The drained property-name tree — flat drop, which also + // dooms its per-axis secondary prefixes when it was indexed. + if *budget == 0 { + return Ok(false); + } + self.grove_drop_flat_subtree( + node_path, + level_key_bytes, + transaction, + drive_operations, + platform_version, + )?; + *budget -= 1; + } + // 2) The terminal `[0]` reference tree, when this level hosts one + // (non-unique / indexOnly layouts; the unique layout stores the + // reference AT key `[0]` as a bare element, which the flat drop of + // this node covers). + let node_path_refs: Vec<&[u8]> = + node_path.iter().map(|segment| segment.as_slice()).collect(); + let zero_element = self.grove_get_raw_optional( + SubtreePath::from(node_path_refs.as_slice()), + &[0], + DirectQueryType::StatefulDirectQuery, + transaction, + drive_operations, + drive_version, + )?; + if let Some(element) = zero_element { + if element.is_any_tree() { + if *budget == 0 { + return Ok(false); + } + self.grove_drop_flat_subtree( + node_path, + &[0], + transaction, + drive_operations, + platform_version, + )?; + *budget -= 1; + } + } + // 3) The node itself. Under an indexed-primary parent grovedb + // refuses every generic child removal (it could not mirror the + // secondary), so the (by now drained) node leaves through the + // dedicated indexed-tree delete matching the parent's tree type — + // which mirrors the ordering value out of the secondary. Under a + // plain parent the flat drop covers any remaining bare elements. + if *budget == 0 { + return Ok(false); + } + let parent_path_refs: Vec<&[u8]> = parent_path + .iter() + .map(|segment| segment.as_slice()) + .collect(); + match parent_tree_type { + TreeType::ProvableCountIndexedTree => { + push_drive_operation_result( + self.grove.delete_from_count_indexed_tree( + SubtreePath::from(parent_path_refs.as_slice()), + node_key, + transaction, + &platform_version.drive.grove_version, + ), + drive_operations, + )?; + } + TreeType::ProvableSumIndexedTree => { + push_drive_operation_result( + self.grove.delete_from_provable_sum_indexed_tree( + SubtreePath::from(parent_path_refs.as_slice()), + node_key, + transaction, + &platform_version.drive.grove_version, + ), + drive_operations, + )?; + } + TreeType::ProvableCountProvableSumIndexedTree => { + push_drive_operation_result( + self.grove + .delete_from_provable_count_provable_sum_indexed_tree( + SubtreePath::from(parent_path_refs.as_slice()), + node_key, + transaction, + &platform_version.drive.grove_version, + ), + drive_operations, + )?; + } + _ => { + self.grove_drop_flat_subtree( + parent_path, + node_key, + transaction, + drive_operations, + platform_version, + )?; + } + } + *budget -= 1; + Ok(true) + } + + /// Cost-pushing wrapper over [`GroveDb::drop_flat_subtree`] — the O(1) + /// consensus detach of a flat subtree with staged prefix reclamation + /// (grovedb#848). Version-gated inside grovedb itself: fail-closed + /// below GROVE_V4, which platform reaches exactly when the TTL grammar + /// exists (protocol v14's drive version carries GROVE_V4). + pub(crate) fn grove_drop_flat_subtree( + &self, + path: &[Vec], + key: &[u8], + transaction: TransactionArg, + drive_operations: &mut Vec, + platform_version: &PlatformVersion, + ) -> Result<(), Error> { + let path_refs: Vec<&[u8]> = path.iter().map(|segment| segment.as_slice()).collect(); + let cost_context = self.grove.drop_flat_subtree( + SubtreePath::from(path_refs.as_slice()), + key, + transaction, + &platform_version.drive.grove_version, + ); + push_drive_operation_result(cost_context, drive_operations) + } +} diff --git a/packages/rs-drive/src/drive/document/update/internal/update_document_for_contract_operations/v1/mod.rs b/packages/rs-drive/src/drive/document/update/internal/update_document_for_contract_operations/v1/mod.rs index 4893094e7ab..c535a4a4df6 100644 --- a/packages/rs-drive/src/drive/document/update/internal/update_document_for_contract_operations/v1/mod.rs +++ b/packages/rs-drive/src/drive/document/update/internal/update_document_for_contract_operations/v1/mod.rs @@ -2,6 +2,7 @@ use crate::drive::constants::CONTRACT_DOCUMENTS_PATH_HEIGHT; use crate::drive::document::index_level_tree_types::{ index_level_tree_types_with_continuation_demotion, IndexLevelTreeTypes, }; +use crate::drive::document::time_range_ttl::{entry_key_bucket_start, live_time_range_entry_keys}; use crate::drive::document::{ make_document_reference, make_document_reference_with_sum_item, read_document_sum_contribution, }; @@ -299,6 +300,35 @@ impl Drive { // beneath a `ProvableCount*` / `ProvableSum*` parent — // diverging from the insert path (consensus break). let index_structure = document_type.index_structure(); + + // TTL drainage rides every write into a TTL'd index — updates + // included, mirroring the v2 insert walker: a bounded number of + // deepest-first drop operations against the oldest expired bucket, + // resuming wherever the previous write's budget ran out. One sweep + // over the deduplicated levels, BEFORE the per-index loop queues + // any batch mutation: drainage applies directly to grovedb, so a + // per-index drain could both multiply the per-write budget (several + // indexes may share one grid level) and remove paths an earlier + // index's queued operations target. Running it first also keeps the + // loop coherent with the drained state: if the drain takes a bucket + // this document's old entries lived in, the old-entry removable + // checks skip it. This path is stateful-only (estimation redirected + // to the insert walker above), and drainage is unbilled — see the + // ttl module's Billing section. + { + let base_path: Vec> = contract_document_type_path + .iter() + .map(|&segment| Vec::from(segment)) + .collect(); + self.drain_expired_time_range_levels( + index_structure, + &base_path, + block_info.time_ms, + transaction, + platform_version, + )?; + } + // fourth we need to store a reference to the document for each index for index in document_type.indexes().values() { // at this point the contract path is to the contract documents @@ -383,6 +413,22 @@ impl Drive { // transform is exactly the grid every index sharing this level // declared.) if let Some(transform) = current_index_level.time_range() { + // TTL'd (ephemeral) sub-levels ride their own op batch and carry + // no storage flags — same routing as the insert and delete + // walkers; see the ttl module's Billing section. + let index_is_ephemeral = transform.ttl_seconds.is_some(); + let mut ephemeral_local_operations: Vec = vec![]; + let index_batch_operations: &mut Vec = if index_is_ephemeral + { + &mut ephemeral_local_operations + } else { + &mut batch_operations + }; + let index_storage_flags = if index_is_ephemeral { + None + } else { + storage_flags + }; self.update_time_range_index_for_contract_operations_v1( index, transform, @@ -393,13 +439,21 @@ impl Drive { &index_path, current_index_level, &index_document_reference, - storage_flags, + index_storage_flags, &mut batch_insertion_cache, previous_batch_operations, - &mut batch_operations, + index_batch_operations, + block_info.time_ms, transaction, platform_version, )?; + if index_is_ephemeral { + batch_operations.extend( + ephemeral_local_operations + .into_iter() + .map(LowLevelDriveOperation::retag_ephemeral), + ); + } continue; } @@ -930,11 +984,18 @@ impl Drive { batch_insertion_cache: &mut HashSet>>, previous_batch_operations: &mut Option<&mut Vec>, batch_operations: &mut Vec, + block_time_ms: u64, transaction: TransactionArg, platform_version: &PlatformVersion, ) -> Result<(), Error> { let drive_version = &platform_version.drive; + // TTL drainage already ran: the caller sweeps every TTL'd level + // once (deduplicated) before the per-index loop, so the removable + // checks below see post-drain state and no direct drop can race a + // queued mutation. Do not drain here — several indexes may share + // this level. + // New/old raw values for the bucketed source property → entry key // sets, mirroring the insert walker's fan-out (see the doc comment). let new_raw = document.get_raw_for_document_type( @@ -960,7 +1021,17 @@ impl Drive { // shared with the insert and delete walkers via // `TimeRangeTransform::entry_keys_for_raw` — one definition, so the // three walkers can never disagree. - let new_entry_keys = transform.entry_keys_for_raw(new_raw.as_deref().unwrap_or_default()); + // TTL: writes never target expired buckets — an update of a document + // whose windows have all expired leaves it with no entries under + // this index, and never resurrects a dropped bucket. The old set is + // NOT filtered: its expired keys flow to the delete loop below, + // whose per-key removable check skips exactly the buckets the lazy + // drop already took. + let new_entry_keys = live_time_range_entry_keys( + transform, + transform.entry_keys_for_raw(new_raw.as_deref().unwrap_or_default()), + block_time_ms, + ); let old_entry_keys = transform.entry_keys_for_raw(old_raw.as_deref().unwrap_or_default()); // Terminator-layout inputs, tracked separately for the new and the old @@ -1254,6 +1325,48 @@ impl Drive { if new_set.contains(entry_key) && !suffix_changed { continue; // unchanged entry — already refreshed by the insert loop above } + // TTL: an expired bucket the drain already took entirely has no + // entry left to delete; one that still stands may be PARTIALLY + // drained (whole `[0]` and group value trees go before the + // bucket), so the entry is checked at full-path granularity + // below and skipped when its deeper trees are gone. A live + // bucket behaves exactly as before. This path is stateful-only + // (estimation redirects to the insert walker at the top of the + // v1 update), so the existence reads are always legal here. + let expired_entry = entry_key_bucket_start(entry_key) + .zip(transform.expiry_horizon_ms(block_time_ms)) + .is_some_and(|(start, horizon)| start < horizon); + if expired_entry { + if !self.time_range_entry_is_removable( + transform, + entry_key, + block_time_ms, + base_index_path, + transaction, + platform_version, + )? { + continue; + } + let mut entry_path_segments: Vec> = base_index_path.to_vec(); + entry_path_segments.push(entry_key.clone()); + for segment in &old_suffix { + entry_path_segments.push(segment.clone()); + } + if !old_terminator_is_unique { + entry_path_segments.push(vec![0]); + } + // `base_index_path` — the document-type path plus the + // grid-qualified level key — exists for every registered + // contract, so the walk starts below it. + if !self.expired_entry_path_exists( + &entry_path_segments, + base_index_path.len(), + transaction, + platform_version, + )? { + continue; + } + } let mut key_info_path: Vec = base_index_path .iter() .map(|s| KnownKey(s.clone())) diff --git a/packages/rs-drive/src/fees/op.rs b/packages/rs-drive/src/fees/op.rs index 7e55a751bd4..67201d28c3a 100644 --- a/packages/rs-drive/src/fees/op.rs +++ b/packages/rs-drive/src/fees/op.rs @@ -7,19 +7,21 @@ use std::collections::BTreeMap; use enum_map::Enum; use grovedb::batch::key_info::KeyInfo; +use grovedb::batch::GroveOp; use grovedb::batch::KeyInfoPath; use grovedb::element::reference_path::ReferencePathType; use grovedb::element::IndexAxis; use grovedb::element::MaxReferenceHop; use grovedb::{batch::QualifiedGroveDbOp, Element, ElementFlags, TreeType}; use grovedb_costs::OperationCost; -use itertools::Itertools; use crate::error::drive::DriveError; +use crate::error::fee::FeeError; use crate::error::Error; use crate::fees::get_overflow_error; use crate::fees::op::LowLevelDriveOperation::{ - CalculatedCostOperation, FunctionOperation, GroveOperation, PreCalculatedFeeResult, + CalculatedCostOperation, CalculatedEphemeralCostOperation, EphemeralGroveOperation, + FunctionOperation, GroveOperation, PreCalculatedFeeResult, }; use crate::util::batch::grovedb_op_batch::GroveDbOpBatchV0Methods; use crate::util::storage_flags::StorageFlags; @@ -205,10 +207,24 @@ impl FunctionOp { pub enum LowLevelDriveOperation { /// Grove operation GroveOperation(QualifiedGroveDbOp), + /// A grove operation targeting a TTL'd `timeRange` index subtree. + /// Applied in its own batch and consumed at the EPHEMERAL price: + /// added bytes bill to processing at the fee table's + /// `ttl_ephemeral_disk_usage_credit_per_byte` instead of to storage + /// (the bytes provably live at most `ttl` plus a bounded drainage + /// lag), and removals produce no refunds — TTL elements carry no + /// storage flags. Produced only by the document index walkers for + /// sub-levels whose transform declares a `ttl`; unreachable before + /// protocol v14, where the grammar does not parse. + EphemeralGroveOperation(QualifiedGroveDbOp), /// A drive operation FunctionOperation(FunctionOp), /// Calculated cost operation CalculatedCostOperation(OperationCost), + /// The applied cost of an ephemeral (TTL'd-subtree) batch — same + /// pricing rule as [`Self::EphemeralGroveOperation`], carrying the + /// cost the batch application (or its estimation) actually returned. + CalculatedEphemeralCostOperation(OperationCost), /// Pre Calculated Fee Result PreCalculatedFeeResult(FeeResult), } @@ -259,6 +275,46 @@ impl LowLevelDriveOperation { processing_fee: op.cost(fee_version), ..Default::default() }), + CalculatedEphemeralCostOperation(cost) => { + // TTL'd-subtree bytes: the added bytes bill to + // PROCESSING at the ephemeral rate instead of to + // storage — they provably live at most `ttl` plus a + // bounded drainage lag, so the perpetual-retention + // storage price does not apply. No refunds by + // construction: TTL elements carry no storage flags, + // so their removal can only ever be basic. + let ephemeral_bytes_fee = (cost.storage_cost.added_bytes as u64) + .checked_mul( + fee_version + .storage + .ttl_ephemeral_disk_usage_credit_per_byte, + ) + .ok_or(Error::Fee(FeeError::Overflow( + "overflow pricing ephemeral bytes", + )))?; + let processing_fee = cost + .ephemeral_cost(fee_version)? + .checked_add(ephemeral_bytes_fee) + .ok_or(Error::Fee(FeeError::Overflow( + "overflow adding ephemeral bytes fee", + )))?; + let removed_bytes_from_system = match cost.storage_cost.removed_bytes { + NoStorageRemoval => 0, + BasicStorageRemoval(amount) => amount, + SectionedStorageRemoval(_) => { + return Err(Error::Drive(DriveError::CorruptedCodeExecution( + "TTL'd subtrees carry no storage flags, so an ephemeral \ + batch cannot produce sectioned (refundable) removal", + ))) + } + }; + Ok(FeeResult { + storage_fee: 0, + processing_fee, + fee_refunds: FeeRefunds::default(), + removed_bytes_from_system, + }) + } _ => { let cost = operation.operation_cost()?; // There is no need for a checked multiply here because added bytes are u64 and @@ -315,10 +371,12 @@ impl LowLevelDriveOperation { /// Returns the cost of this operation pub fn operation_cost(self) -> Result { match self { - GroveOperation(_) => Err(Error::Drive(DriveError::CorruptedCodeExecution( - "grove operations must be executed, not directly transformed to costs", - ))), - CalculatedCostOperation(c) => Ok(c), + GroveOperation(_) | EphemeralGroveOperation(_) => { + Err(Error::Drive(DriveError::CorruptedCodeExecution( + "grove operations must be executed, not directly transformed to costs", + ))) + } + CalculatedCostOperation(c) | CalculatedEphemeralCostOperation(c) => Ok(c), PreCalculatedFeeResult(_) => Err(Error::Drive(DriveError::CorruptedCodeExecution( "pre calculated fees should not be requested by operation costs", ))), @@ -371,18 +429,70 @@ impl LowLevelDriveOperation { pub fn grovedb_operations_batch_consume_with_leftovers( insert_operations: Vec, ) -> (GroveDbOpBatch, Vec) { - let (grove_operations, other_operations): (Vec<_>, Vec<_>) = - insert_operations.into_iter().partition_map(|op| match op { - GroveOperation(grovedb_op) => itertools::Either::Left(grovedb_op), - _ => itertools::Either::Right(op), - }); + let (batch, ephemeral_batch, other_operations) = + Self::grovedb_operations_batch_consume_split_ephemeral(insert_operations); + debug_assert!( + ephemeral_batch.is_empty(), + "ephemeral grove operations must go through the ephemeral-aware apply" + ); + (batch, other_operations) + } + /// Splits operations three ways: the ordinary grove batch, the + /// ephemeral (TTL'd-subtree) grove batch — applied separately so its + /// cost can be consumed at the ephemeral price — and every + /// non-grove leftover. + pub fn grovedb_operations_batch_consume_split_ephemeral( + insert_operations: Vec, + ) -> (GroveDbOpBatch, GroveDbOpBatch, Vec) { + let mut grove_operations = vec![]; + let mut ephemeral_operations = vec![]; + let mut other_operations = vec![]; + for op in insert_operations { + match op { + GroveOperation(grovedb_op) => grove_operations.push(grovedb_op), + EphemeralGroveOperation(grovedb_op) => ephemeral_operations.push(grovedb_op), + other => other_operations.push(other), + } + } ( GroveDbOpBatch::from_operations(grove_operations), + GroveDbOpBatch::from_operations(ephemeral_operations), other_operations, ) } + /// Re-tag an operation as targeting a TTL'd (ephemeral) subtree, so + /// its bytes are consumed at the ephemeral price. Grove operations + /// move to their ephemeral batch; already-calculated costs keep their + /// numbers under the ephemeral consumption rule; fee results pass + /// through untouched (nothing byte-priced remains in them). + pub fn retag_ephemeral(self) -> LowLevelDriveOperation { + match self { + GroveOperation(mut grovedb_op) => { + // TTL'd (ephemeral) subtrees must hold flagless elements: + // their bytes are never refundable, and flags on any element + // under them would turn its later removal sectioned + // (refundable) — the consume path treats that as corruption. + // Stripping here, at the single choke point every ephemeral + // op passes through, lets the walkers keep building elements + // exactly as they do for standing levels. + match &mut grovedb_op.op { + GroveOp::InsertWithKnownToNotAlreadyExist { element } + | GroveOp::InsertIfNotExists { element, .. } + | GroveOp::InsertOrReplace { element } + | GroveOp::Replace { element } + | GroveOp::Patch { element, .. } => element.set_flags(None), + GroveOp::RefreshReference { flags, .. } => *flags = None, + _ => {} + } + EphemeralGroveOperation(grovedb_op) + } + CalculatedCostOperation(cost) => CalculatedEphemeralCostOperation(cost), + other => other, + } + } + /// Filters the groveDB ops from a list of operations and collects them in a `Vec`. pub fn grovedb_operations_consume( insert_operations: Vec, diff --git a/packages/rs-drive/src/query/drive_document_average_query/drive_dispatcher.rs b/packages/rs-drive/src/query/drive_document_average_query/drive_dispatcher.rs index da9f8de2de6..9cadac3340b 100644 --- a/packages/rs-drive/src/query/drive_document_average_query/drive_dispatcher.rs +++ b/packages/rs-drive/src/query/drive_document_average_query/drive_dispatcher.rs @@ -2029,6 +2029,7 @@ mod tests { range_seconds: 21_600, step_seconds: 7_200, phase_seconds: 0, + ttl_seconds: None, }, }], }; diff --git a/packages/rs-drive/src/query/drive_document_count_query/tests.rs b/packages/rs-drive/src/query/drive_document_count_query/tests.rs index 107e5bd4bb7..2a39d8f0f08 100644 --- a/packages/rs-drive/src/query/drive_document_count_query/tests.rs +++ b/packages/rs-drive/src/query/drive_document_count_query/tests.rs @@ -3566,6 +3566,7 @@ mod time_range_picker_tests { range_seconds: 6 * HOUR_SECONDS, step_seconds: 2 * HOUR_SECONDS, phase_seconds: 0, + ttl_seconds: None, }), ), ] @@ -3585,6 +3586,7 @@ mod time_range_picker_tests { range_seconds: 6 * HOUR_SECONDS, step_seconds: 2 * HOUR_SECONDS, phase_seconds: 0, + ttl_seconds: None, }, }] } diff --git a/packages/rs-drive/src/query/mod.rs b/packages/rs-drive/src/query/mod.rs index f154d2fd128..ea4c87f16e6 100644 --- a/packages/rs-drive/src/query/mod.rs +++ b/packages/rs-drive/src/query/mod.rs @@ -821,9 +821,11 @@ impl ResolvedTimeRange { /// response metadata `time_ms`, so both produce the identical concrete /// equality query — the existing index/count proofs apply unchanged and the /// engine never needs a dedicated time-range operator. A -/// [`TimeRangeSelector::ByStart`] selection ignores `block_time_ms` -/// entirely: the start is in the query itself (validated to lie on the -/// grid), so both sides read the same window with no clock involved. +/// [`TimeRangeSelector::ByStart`] selection reads its start from the query +/// itself (validated to lie on the grid) and consults `block_time_ms` only +/// to reject windows past a declared `ttl`'s horizon — a window that may be +/// mid-drainage must not serve a truncated answer, and since drainage only +/// touches expired buckets, every window this resolver admits is complete. /// /// `grid` selects among several time-range indexes on the same field: `None` /// is accepted only while exactly one grid buckets the field (the common @@ -893,8 +895,9 @@ pub fn resolve_time_range_bucket_clause( let bucket_start = match selector { TimeRangeSelector::Newest => transform.newest_active_start(block_time_ms), TimeRangeSelector::Oldest => transform.oldest_active_start(block_time_ms), - // Absolute selection: the start comes from the query itself, so no - // clock is consulted — prover and verifier agree by construction. + // Absolute selection: the start comes from the query itself, so the + // clock is consulted only for the TTL gate — prover and verifier + // agree by construction (the verifier passes the signed time_ms). // Only grid membership is checked; an empty (or not-yet-started) // window is a provable empty answer, not an invalid question. TimeRangeSelector::ByStart { start_ms } => { @@ -910,6 +913,25 @@ pub fn resolve_time_range_bucket_clause( transform.phase_seconds )))); } + // TTL gate: an expired window may be mid-drainage, and a + // partially drained window would serve a truncated answer that + // looks authoritative. Drainage only ever touches expired + // buckets (same `bucket_expired` predicate), so everything on + // the queryable side of this gate is complete — and rejecting + // the question is deterministic where "whatever the drain has + // left" is not. The verifier resolves this clause with the + // quorum-signed response `time_ms`, so a node cannot serve an + // expired window's remnants past a verifying client. + if transform.bucket_expired(start_ms, block_time_ms) { + return Err(Error::Query(QuerySyntaxError::Unsupported(format!( + "byStart {} on \"{}\" is past the ttl horizon ({}s): expired windows \ + drain lazily and may be mid-removal, so they are not queryable — \ + entries under this index live at most `ttl` past their window's start", + start_ms, + field, + transform.ttl_seconds.unwrap_or_default() + )))); + } Some(start_ms) } } @@ -3601,6 +3623,7 @@ mod tests { range_seconds: 21_600, step_seconds: 7_200, phase_seconds: 0, + ttl_seconds: None, }, }]; let equality = WhereClause { @@ -3639,6 +3662,115 @@ mod tests { .expect_err("a resolved field with no equality at all must be rejected"); } + /// The TTL horizon gate: an expired window may be mid-drainage, so + /// `byStart` must reject it rather than serve whatever the drain has + /// left. The boundary is the drain's own predicate — a window starting + /// exactly at the horizon is not yet expired and stays queryable — and + /// an index without a `ttl` keeps serving arbitrarily old windows. + #[test] + fn by_start_rejects_windows_past_the_ttl_horizon() { + use crate::query::{resolve_time_range_bucket_clause, TimeRangeSelector}; + use dpp::data_contract::DataContractFactory; + use dpp::platform_value::platform_value; + use dpp::prelude::Identifier; + + let factory = + DataContractFactory::new(PlatformVersion::latest().protocol_version).expect("factory"); + let hour_ms: u64 = 3_600_000; + let build = |seed: u8, with_ttl: bool| { + let mut time_range = vec![ + ( + Value::Text("on".to_string()), + Value::Text("$createdAt".to_string()), + ), + (Value::Text("range".to_string()), Value::U64(7_200)), + (Value::Text("step".to_string()), Value::U64(7_200)), + ]; + if with_ttl { + time_range.push((Value::Text("ttl".to_string()), Value::U64(14_400))); + } + let index_map = vec![ + ( + Value::Text("name".to_string()), + Value::Text("trending".to_string()), + ), + ( + Value::Text("properties".to_string()), + Value::Array(vec![ + platform_value!({"$createdAt": "asc"}), + platform_value!({"hashtag": "asc"}), + ]), + ), + (Value::Text("timeRange".to_string()), Value::Map(time_range)), + ( + Value::Text("countable".to_string()), + Value::Text("countable".to_string()), + ), + ]; + let document_schema = platform_value!({ + "type": "object", + "properties": { + "hashtag": {"type": "string", "maxLength": 61, "position": 0}, + }, + "required": ["hashtag", "$createdAt"], + "indices": Value::Array(vec![Value::Map(index_map)]), + "additionalProperties": false, + }); + factory + .create_with_value_config( + Identifier::from([seed; 32]), + 0, + platform_value!({ "post": document_schema }), + None, + None, + ) + .expect("contract registers") + .data_contract_owned() + }; + + let ttl_contract = build(101, true); + let standing_contract = build(102, false); + let expired_start = 5_000 * hour_ms; + let block_time = expired_start + 6 * hour_ms; + + let resolve = |contract: &DataContract, start_ms: u64| { + resolve_time_range_bucket_clause( + "$createdAt", + TimeRangeSelector::ByStart { start_ms }, + None, + contract + .document_type_for_name("post") + .expect("document type"), + block_time, + ) + }; + + let error = resolve(&ttl_contract, expired_start) + .expect_err("a window past the ttl horizon must be rejected, not served"); + assert!( + error.to_string().contains("ttl horizon"), + "the rejection names the horizon: {error}" + ); + resolve(&ttl_contract, expired_start + 2 * hour_ms).expect( + "a window starting exactly at the horizon is not expired — same \ + strictly-below boundary the drain uses", + ); + resolve(&ttl_contract, expired_start + 4 * hour_ms) + .expect("a live window resolves normally"); + resolve_time_range_bucket_clause( + "$createdAt", + TimeRangeSelector::Newest, + None, + ttl_contract + .document_type_for_name("post") + .expect("document type"), + block_time, + ) + .expect("relative selectors never address expired windows and stay unaffected"); + resolve(&standing_contract, expired_start) + .expect("without a ttl, arbitrarily old windows stay queryable"); + } + #[test] fn test_withdrawal_query_with_missing_transaction_index() { // Setup the withdrawal contract diff --git a/packages/rs-drive/src/util/batch/drive_op_batch/document.rs b/packages/rs-drive/src/util/batch/drive_op_batch/document.rs index 3b996ef0cfd..7e7f798d1e4 100644 --- a/packages/rs-drive/src/util/batch/drive_op_batch/document.rs +++ b/packages/rs-drive/src/util/batch/drive_op_batch/document.rs @@ -335,6 +335,7 @@ impl DriveLowLevelOperationConverter for DocumentOperationType<'_> { document_type, None, estimated_costs_only_with_layer_info, + block_info.time_ms, transaction, platform_version, ) @@ -366,6 +367,7 @@ impl DriveLowLevelOperationConverter for DocumentOperationType<'_> { document_type, None, estimated_costs_only_with_layer_info, + block_info.time_ms, transaction, platform_version, ) diff --git a/packages/rs-drive/src/util/grove_operations/mod.rs b/packages/rs-drive/src/util/grove_operations/mod.rs index ef3b826c99e..2535798c930 100644 --- a/packages/rs-drive/src/util/grove_operations/mod.rs +++ b/packages/rs-drive/src/util/grove_operations/mod.rs @@ -226,7 +226,7 @@ use intmap::IntMap; /// Pushes an operation's `OperationCost` to `drive_operations` given its `CostContext` /// and returns the operation's return value. -fn push_drive_operation_result( +pub(crate) fn push_drive_operation_result( cost_context: CostContext>, drive_operations: &mut Vec, ) -> Result { diff --git a/packages/rs-drive/src/util/operations/apply_batch_low_level_drive_operations/v0/mod.rs b/packages/rs-drive/src/util/operations/apply_batch_low_level_drive_operations/v0/mod.rs index 4367f56b14b..da56d5bc958 100644 --- a/packages/rs-drive/src/util/operations/apply_batch_low_level_drive_operations/v0/mod.rs +++ b/packages/rs-drive/src/util/operations/apply_batch_low_level_drive_operations/v0/mod.rs @@ -20,10 +20,22 @@ impl Drive { drive_operations: &mut Vec, drive_version: &DriveVersion, ) -> Result<(), Error> { - let (grove_db_operations, mut other_operations) = - LowLevelDriveOperation::grovedb_operations_batch_consume_with_leftovers( + let (grove_db_operations, ephemeral_grove_db_operations, mut other_operations) = + LowLevelDriveOperation::grovedb_operations_batch_consume_split_ephemeral( batch_operations, ); + // The ephemeral (TTL'd-subtree) operations apply as their own batch + // so their cost is known separately and can be consumed at the + // ephemeral price — added bytes to processing instead of storage. + // Cloning the layer info keeps the estimation path symmetric: the + // dry run prices the ephemeral batch through the same worst-case + // machinery, under the same pricing rule, so estimated stays an + // upper bound of actual per fee class. + let ephemeral_layer_info = if ephemeral_grove_db_operations.is_empty() { + None + } else { + estimated_costs_only_with_layer_info.clone() + }; if !grove_db_operations.is_empty() { self.apply_batch_grovedb_operations( estimated_costs_only_with_layer_info, @@ -33,6 +45,21 @@ impl Drive { drive_version, )?; } + if !ephemeral_grove_db_operations.is_empty() { + let mut ephemeral_cost_operations: Vec = vec![]; + self.apply_batch_grovedb_operations( + ephemeral_layer_info, + transaction, + ephemeral_grove_db_operations, + &mut ephemeral_cost_operations, + drive_version, + )?; + drive_operations.extend( + ephemeral_cost_operations + .into_iter() + .map(LowLevelDriveOperation::retag_ephemeral), + ); + } drive_operations.append(&mut other_operations); Ok(()) } diff --git a/packages/rs-platform-version/Cargo.toml b/packages/rs-platform-version/Cargo.toml index 8046567e830..7226e389536 100644 --- a/packages/rs-platform-version/Cargo.toml +++ b/packages/rs-platform-version/Cargo.toml @@ -11,7 +11,7 @@ license = "MIT" thiserror = { version = "2.0.12" } bincode = { version = "=2.0.1" } versioned-feature-core = { git = "https://github.com/dashpay/versioned-feature-core", version = "1.0.0" } -grovedb-version = { git = "https://github.com/dashpay/grovedb", rev = "97250247423ddc7fcc2865ea0025fc5bd41d0883" } +grovedb-version = { git = "https://github.com/dashpay/grovedb", rev = "d548e28228e2aca1361d9b12835388be0f811a92" } [features] mock-versions = [] diff --git a/packages/rs-platform-version/src/version/fee/mod.rs b/packages/rs-platform-version/src/version/fee/mod.rs index 8b603e2ddba..b4b6f0554a5 100644 --- a/packages/rs-platform-version/src/version/fee/mod.rs +++ b/packages/rs-platform-version/src/version/fee/mod.rs @@ -85,11 +85,41 @@ impl FeeVersion { // The issue was that the platform state was stored with FeeVersions in it before version 1.4 // When we would add new fields we would be unable to deserialize // This FeeProcessingVersionFieldsBeforeVersion4 is how things were before version 1.4 was released +/// The storage fee table exactly as every pre-4.2 release serialized it +/// (5 fields). `FeeStorageVersion` gained +/// `ttl_ephemeral_disk_usage_credit_per_byte` in 4.2; embedding the live +/// struct here would shift the frozen pre-1.4 platform-state wire format +/// and break deserialization of old stored states (bincode +/// `UnexpectedEnd`). Any future field added to `FeeStorageVersion` must +/// NOT be added here. +#[derive(Clone, Debug, Encode, Decode, Default, PartialEq, Eq)] +pub struct FeeStorageVersionFieldsBeforeVersion4 { + pub storage_disk_usage_credit_per_byte: u64, + pub storage_processing_credit_per_byte: u64, + pub storage_load_credit_per_byte: u64, + pub non_storage_load_credit_per_byte: u64, + pub storage_seek_cost: u64, +} + +impl From for FeeStorageVersion { + fn from(value: FeeStorageVersionFieldsBeforeVersion4) -> Self { + FeeStorageVersion { + storage_disk_usage_credit_per_byte: value.storage_disk_usage_credit_per_byte, + storage_processing_credit_per_byte: value.storage_processing_credit_per_byte, + storage_load_credit_per_byte: value.storage_load_credit_per_byte, + non_storage_load_credit_per_byte: value.non_storage_load_credit_per_byte, + storage_seek_cost: value.storage_seek_cost, + // Pre-4.2 tables predate the TTL grammar entirely. + ttl_ephemeral_disk_usage_credit_per_byte: 0, + } + } +} + #[derive(Clone, Debug, Encode, Decode, Default, PartialEq, Eq)] pub struct FeeVersionFieldsBeforeVersion4 { // Permille means devise by 1000 pub uses_version_fee_multiplier_permille: Option, - pub storage: FeeStorageVersion, + pub storage: FeeStorageVersionFieldsBeforeVersion4, pub signature: FeeSignatureVersion, pub hashing: FeeHashingVersionBeforeVersion11, pub processing: FeeProcessingVersionFieldsBeforeVersion1Point4, @@ -103,7 +133,7 @@ impl From for FeeVersion { FeeVersion { fee_version_number: 1, uses_version_fee_multiplier_permille: value.uses_version_fee_multiplier_permille, - storage: value.storage, + storage: value.storage.into(), signature: value.signature, hashing: FEE_HASHING_VERSION1, processing: FeeProcessingVersion::from(value.processing), diff --git a/packages/rs-platform-version/src/version/fee/storage/mod.rs b/packages/rs-platform-version/src/version/fee/storage/mod.rs index f6121321ba5..0e2ffa41969 100644 --- a/packages/rs-platform-version/src/version/fee/storage/mod.rs +++ b/packages/rs-platform-version/src/version/fee/storage/mod.rs @@ -9,6 +9,14 @@ pub struct FeeStorageVersion { pub storage_load_credit_per_byte: u64, pub non_storage_load_credit_per_byte: u64, pub storage_seek_cost: u64, + /// Credits charged per byte written under a TTL'd `timeRange` index + /// subtree, billed to PROCESSING in place of the storage fee: the + /// bytes provably live at most one week (the `ttl` cap) plus a + /// bounded drainage lag, so charging them the perpetual-retention + /// storage price would overprice them by orders of magnitude. Zero + /// until protocol version 14 — the `ttl` grammar does not parse + /// before it, so no ephemeral-classified operation can exist. + pub ttl_ephemeral_disk_usage_credit_per_byte: u64, } #[cfg(test)] @@ -24,6 +32,7 @@ mod tests { storage_load_credit_per_byte: 3, non_storage_load_credit_per_byte: 4, storage_seek_cost: 5, + ttl_ephemeral_disk_usage_credit_per_byte: 6, }; let version2 = FeeStorageVersion { @@ -32,9 +41,21 @@ mod tests { storage_load_credit_per_byte: 3, non_storage_load_credit_per_byte: 4, storage_seek_cost: 5, + ttl_ephemeral_disk_usage_credit_per_byte: 6, }; // This assertion will check if all fields are considered in the equality comparison assert_eq!(version1, version2, "FeeStorageVersion equality test failed. If a field was added or removed, update the Eq implementation."); + + // And the inequality direction: a difference in the newest field + // alone must be visible to Eq. + let version3 = FeeStorageVersion { + ttl_ephemeral_disk_usage_credit_per_byte: 7, + ..version2.clone() + }; + assert_ne!( + version1, version3, + "FeeStorageVersion equality must distinguish ttl_ephemeral_disk_usage_credit_per_byte" + ); } } diff --git a/packages/rs-platform-version/src/version/fee/storage/v1.rs b/packages/rs-platform-version/src/version/fee/storage/v1.rs index dcc30327ac4..ec3e0605d61 100644 --- a/packages/rs-platform-version/src/version/fee/storage/v1.rs +++ b/packages/rs-platform-version/src/version/fee/storage/v1.rs @@ -8,4 +8,17 @@ pub const FEE_STORAGE_VERSION1: FeeStorageVersion = FeeStorageVersion { storage_load_credit_per_byte: 20, non_storage_load_credit_per_byte: 10, storage_seek_cost: 2000, + // Dead below protocol v14: the `ttl` grammar does not parse there, so + // no ephemeral-classified operation can exist and nothing reads this + // rate — carrying the live value here changes no released behavior. + // + // The rate prices a byte that provably lives at most one week (the + // `ttl` cap) plus a bounded drainage lag. Pro-rata against the + // perpetual-retention price (27,000 credits/byte distributed over ~50 + // years) one week is ~10 credits/byte; 270 — one percent of the + // storage price — keeps a ~27x margin for the drainage work the + // triggering writes perform unbilled and for disk churn, while still + // making windowed (trending) writes two orders of magnitude cheaper + // than permanent ones. + ttl_ephemeral_disk_usage_credit_per_byte: 270, }; diff --git a/packages/rs-platform-version/src/version/mocks/v2_test.rs b/packages/rs-platform-version/src/version/mocks/v2_test.rs index 76579f3a0d7..125fd24356a 100644 --- a/packages/rs-platform-version/src/version/mocks/v2_test.rs +++ b/packages/rs-platform-version/src/version/mocks/v2_test.rs @@ -516,6 +516,8 @@ pub const TEST_PLATFORM_V2: PlatformVersion = PlatformVersion { max_token_redemption_cycles: 128, max_shielded_transition_actions: 16, max_time_range_overlap_factor: None, + max_time_range_ttl_seconds: None, + max_time_range_ttl_drop_operations_per_write: None, }, consensus: ConsensusVersions { tenderdash_consensus_version: 0, diff --git a/packages/rs-platform-version/src/version/system_limits/mod.rs b/packages/rs-platform-version/src/version/system_limits/mod.rs index 0854a6c6f3b..af9c38ed962 100644 --- a/packages/rs-platform-version/src/version/system_limits/mod.rs +++ b/packages/rs-platform-version/src/version/system_limits/mod.rs @@ -100,6 +100,30 @@ pub struct SystemLimits { /// time-range indexes (nothing to bound: the `timeRange` keyword does not /// parse there). pub max_time_range_overlap_factor: Option, + /// Maximum time-to-live (in seconds) a `timeRange` index transform may + /// declare, enforced at contract registration. + /// + /// The cap is what makes the TTL fee model safe: entries under a TTL'd + /// index bill their bytes as processing (the ephemeral-bytes rate) + /// instead of storage, and a flat rate is only an honest price while + /// the lifetime it covers is bounded. One week in V4. + /// See `book/src/drive/time-range-ttl.md`. + /// + /// `None` preserves the behavior of protocol versions that predate the + /// `ttl` key (nothing to bound: the key does not parse there). + pub max_time_range_ttl_seconds: Option, + /// Maximum number of O(1) drop operations one write into a TTL'd + /// `timeRange` index may spend draining expired buckets. + /// + /// A bucket drains deepest-first through flat-subtree drops (one per + /// `[0]` reference tree, per emptied value tree, per property-name + /// tree, plus the bucket itself), so the operation count scales with + /// the window's distinct groups while each operation is O(1). Every + /// write continues wherever the previous budget ran out; write volume + /// scales with group volume, so drainage keeps pace roughly one window + /// behind. `None` for the protocol versions that predate the `ttl` + /// key. + pub max_time_range_ttl_drop_operations_per_write: Option, } #[cfg(test)] diff --git a/packages/rs-platform-version/src/version/system_limits/v1.rs b/packages/rs-platform-version/src/version/system_limits/v1.rs index 57f51ca806f..4d57d43d4b7 100644 --- a/packages/rs-platform-version/src/version/system_limits/v1.rs +++ b/packages/rs-platform-version/src/version/system_limits/v1.rs @@ -50,4 +50,6 @@ pub const SYSTEM_LIMITS_V1: SystemLimits = SystemLimits { // `seed_pool_batch_fits_max_state_transition_size` signing test. max_shielded_transition_actions: 16, max_time_range_overlap_factor: None, + max_time_range_ttl_seconds: None, + max_time_range_ttl_drop_operations_per_write: None, }; diff --git a/packages/rs-platform-version/src/version/system_limits/v2.rs b/packages/rs-platform-version/src/version/system_limits/v2.rs index 3b3f46d5e43..82be9f39f70 100644 --- a/packages/rs-platform-version/src/version/system_limits/v2.rs +++ b/packages/rs-platform-version/src/version/system_limits/v2.rs @@ -31,4 +31,6 @@ pub const SYSTEM_LIMITS_V2: SystemLimits = SystemLimits { // `seed_pool_batch_fits_max_state_transition_size` signing test. max_shielded_transition_actions: 16, max_time_range_overlap_factor: None, + max_time_range_ttl_seconds: None, + max_time_range_ttl_drop_operations_per_write: None, }; diff --git a/packages/rs-platform-version/src/version/system_limits/v3.rs b/packages/rs-platform-version/src/version/system_limits/v3.rs index cdf3248de17..ccb8c536676 100644 --- a/packages/rs-platform-version/src/version/system_limits/v3.rs +++ b/packages/rs-platform-version/src/version/system_limits/v3.rs @@ -33,4 +33,6 @@ pub const SYSTEM_LIMITS_V3: SystemLimits = SystemLimits { // `seed_pool_batch_fits_max_state_transition_size` signing test. max_shielded_transition_actions: 16, max_time_range_overlap_factor: None, + max_time_range_ttl_seconds: None, + max_time_range_ttl_drop_operations_per_write: None, }; diff --git a/packages/rs-platform-version/src/version/system_limits/v4.rs b/packages/rs-platform-version/src/version/system_limits/v4.rs index efc8d72578e..f08b4605741 100644 --- a/packages/rs-platform-version/src/version/system_limits/v4.rs +++ b/packages/rs-platform-version/src/version/system_limits/v4.rs @@ -1,8 +1,22 @@ use crate::version::system_limits::SystemLimits; -/// System limits for protocol version 14 and above. +/// System limits for protocol version 14 and above. Relative to the last +/// released table (V3) this changes the withdrawal limit, adds the +/// time-range overlap-factor cap, and adds the time-range TTL pair +/// (the TTL fields joined this still-unreleased table in place rather +/// than spawning a new version): /// -/// Identical to [`super::v3::SYSTEM_LIMITS_V3`] except for two changes: +/// * `max_time_range_ttl_seconds` is set to one week: the ceiling on the +/// `ttl` a `timeRange` index transform may declare. The cap is what makes +/// the ephemeral-bytes fee model safe — a flat processing rate is only an +/// honest price for transitional storage while the lifetime it covers is +/// bounded. See `book/src/drive/time-range-ttl.md`. +/// * `max_time_range_ttl_drop_operations_per_write` is set to 32: each +/// write into a TTL'd index spends at most this many O(1) flat-drop +/// operations draining expired buckets, deepest-first, resuming across +/// writes. +/// +/// The withdrawal and overlap-factor changes: /// /// * The daily withdrawal limit becomes relative: `daily_withdrawal_limit_percent` is set to 15, /// so Platform pools at most 15% of the total credits it held a day ago into asset unlock @@ -41,4 +55,6 @@ pub const SYSTEM_LIMITS_V4: SystemLimits = SystemLimits { // `seed_pool_batch_fits_max_state_transition_size` signing test. max_shielded_transition_actions: 16, max_time_range_overlap_factor: Some(24), + max_time_range_ttl_seconds: Some(604_800), // one week + max_time_range_ttl_drop_operations_per_write: Some(32), }; diff --git a/packages/rs-platform-version/src/version/v14.rs b/packages/rs-platform-version/src/version/v14.rs index 7589c485738..a3687f1bfea 100644 --- a/packages/rs-platform-version/src/version/v14.rs +++ b/packages/rs-platform-version/src/version/v14.rs @@ -222,8 +222,11 @@ pub const PLATFORM_V14: PlatformVersion = PlatformVersion { factory_versions: DPP_FACTORY_VERSIONS_V1, }, system_data_contracts: SYSTEM_DATA_CONTRACT_VERSIONS_V3, // changed: DashPay v2 adds profile payment address fields (DIP-33) + // The TTL ephemeral-bytes rate (270 credits/byte to processing) rides + // the shared storage table; it is dead below v14 (the `ttl` grammar + // does not parse), so no table fork is needed. fee_version: FEE_VERSION2, - system_limits: SYSTEM_LIMITS_V4, // changed: daily withdrawal limit becomes 15% of the total credits a day ago + time-range overlap-factor cap (24) + system_limits: SYSTEM_LIMITS_V4, // changed: daily withdrawal 15% of total credits a day ago + time-range overlap-factor cap (24) + time-range TTL cap (1 week) and per-write drop cap (32) consensus: ConsensusVersions { tenderdash_consensus_version: 1, }, diff --git a/packages/rs-platform-wallet/Cargo.toml b/packages/rs-platform-wallet/Cargo.toml index 20899d523a3..100c27f31d5 100644 --- a/packages/rs-platform-wallet/Cargo.toml +++ b/packages/rs-platform-wallet/Cargo.toml @@ -69,7 +69,7 @@ zeroize = "1" log = "0.4" # Shielded pool (optional, behind `shielded` feature) -grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "97250247423ddc7fcc2865ea0025fc5bd41d0883", optional = true } +grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "d548e28228e2aca1361d9b12835388be0f811a92", optional = true } # Direct `rusqlite` access so `FileBackedShieldedStore::open_path` can set # WAL + synchronous=NORMAL pragmas before handing the connection to # `ClientPersistentCommitmentTree`. Version locked to match the rev grovedb diff --git a/packages/rs-sdk/Cargo.toml b/packages/rs-sdk/Cargo.toml index 8b71ddbf22b..4f598c4dfd4 100644 --- a/packages/rs-sdk/Cargo.toml +++ b/packages/rs-sdk/Cargo.toml @@ -18,7 +18,7 @@ drive = { path = "../rs-drive", default-features = false, features = [ ] } drive-proof-verifier = { path = "../rs-drive-proof-verifier", default-features = false } -grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "97250247423ddc7fcc2865ea0025fc5bd41d0883", features = [ +grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "d548e28228e2aca1361d9b12835388be0f811a92", features = [ "client", "sqlite", ], optional = true }