From dd4de65e447a305440c4df25e9381aa0ab99df6b Mon Sep 17 00:00:00 2001 From: rbitcoin-grok Date: Sat, 12 Sep 2026 20:05:10 -0700 Subject: [PATCH] store: spent-range from RAM n_out only Lookup was reconstructing spent.body (off,len) by preading txout.body to recover n_out. Class A already has n_out at append; keep that vec and make tx_spent_range* fail closed if it is missing instead of walking the body. Co-authored-by: Cursor --- CHANGELOG.md | 4 + SCHEMA.md | 4 +- crates/rbitcoin-store/src/tx_table/mod.rs | 19 +- .../rbitcoin-store/src/tx_table/spent_off.rs | 179 +++++++++++------- crates/rbitcoin-store/src/tx_table/tests.rs | 23 +++ docs/invariants.md | 2 +- 6 files changed, 153 insertions(+), 78 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2996e81..77bc787c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,10 @@ before 1.0). ### Fixed +- **Spent-range never reads `txout.body`:** `tx_spent_range*` prefix-sums RAM + `n_out` from Class A append (open hydrates that vec from LAYOUT17 meta). + Missing `n_out` is `Corrupt("invariant: spent n_out missing")`. + - **Compact reconstruct merkle-checks before `Ok`:** a unique short-id (or `blocktxn`) fill is not a block until the txs match the compact header merkle (BIP152 `FinishBlock`). Empty missing → `getdata`, not diff --git a/SCHEMA.md b/SCHEMA.md index 7c332cef..72ea48b6 100644 --- a/SCHEMA.md +++ b/SCHEMA.md @@ -2,7 +2,7 @@ **Version:** `SCHEMA_VERSION = 21` (`rbitcoin_primitives`). **Status:** 21 drops `spent.idx`. Spent `(off,len)` is `8 × max(n_out,1)` from -txout meta; sparse `spent.off` (u64/1024 creates). A 21 binary unlinks leftover +RAM `n_out` (sparse `spent.off`, u64/1024 creates). A 21 binary unlinks leftover `spent.idx` (dir and flat `spent.idx.meta`) and rewrites `store/meta` 20→21. A 20 binary refuses 21 `meta`. Occupied schema 18/19 `tx.head` or `scripthash*` is **refused** (wipe those index dirs, keep Class A). Empty @@ -361,7 +361,7 @@ i = fk - first_fk Hard span per segment: `2^32 × 8` ≈ 32 GiB. Soft rollover earlier (default 16 GiB; `RBITCOIN_TX_IDX_SOFT_SPAN`). **Each stem rolls independently** when that stem’s next start would exceed the soft span (`inwit` no longer forces `txout` idx splits). Length: `start(fk+1) − start(fk)` (may cross segments); last record uses published body end. ~**4 B/tx** vs prior 8 B absolute u64 index (~50% smaller). -**`spent.body` has no `spent.idx`.** Record length is `8 × max(n_out, 1)` (zero-out still pays one stride so starts stay monotone). `spent_abs(off, vout) = off + 8×vout`. `n_out` is txout LAYOUT17 meta. Sparse `spent.off` stores absolute starts every 1024 creates (`ArrayLink` u64 LE). Leftover `spent.idx/` (and flat `spent.idx.meta`) is unlinked on open; `store/meta` is rewritten to 21. +**`spent.body` has no `spent.idx`.** Record length is `8 × max(n_out, 1)` (zero-out still pays one stride so starts stay monotone). `spent_abs(off, vout) = off + 8×vout`. `n_out` is Class A append RAM. Sparse `spent.off` stores absolute starts every 1024 creates (`ArrayLink` u64 LE). Spent-range APIs prefix-sum that vec and never read `txout.body`. Open hydrates the vec from LAYOUT17 meta once. Leftover `spent.idx/` (and flat `spent.idx.meta`) is unlinked on open; `store/meta` is rewritten to 21. ### Input encoding (embedded) diff --git a/crates/rbitcoin-store/src/tx_table/mod.rs b/crates/rbitcoin-store/src/tx_table/mod.rs index d040fea0..87e85221 100644 --- a/crates/rbitcoin-store/src/tx_table/mod.rs +++ b/crates/rbitcoin-store/src/tx_table/mod.rs @@ -660,7 +660,7 @@ impl TxTable { } let n_bodies = body.count(); spent_off.ensure_covering(&body, n_bodies)?; - let spent_end = spent_off.end_for(&body, n_bodies)?; + let spent_end = spent_off.end_for(n_bodies)?; if spent.body_logical_len() < spent_end { return Err(StoreError::Corrupt("spent.body short for n_out prefix")); } @@ -1050,15 +1050,14 @@ impl TxTable { self.body.record_range_batch(fks) } - /// `spent.body` range for one create. + /// `spent.body` range for one create (RAM `n_out` + `spent.off`; no `txout.body`). pub fn spent_range(&self, fk: Fk) -> Result<(u64, u64), StoreError> { - self.spent_off.range_for(&self.body, fk, self.spent.count()) + self.spent_off.range_for(fk, self.spent.count()) } /// `spent.body` ranges (same fk order as [`Self::body_range_batch`]). pub fn spent_range_batch(&self, fks: &[Fk]) -> Result>, StoreError> { - self.spent_off - .ranges_batch(&self.body, fks, self.spent.count()) + self.spent_off.ranges_batch(fks, self.spent.count()) } /// Annotate spends at known absolute spender-meta offsets (confirm write). @@ -1516,6 +1515,7 @@ impl TxTable { if self.inwit.count() != base || self.spent.count() != base { return Err(StoreError::Corrupt("Class A stem count mismatch on append")); } + let n_outs: Vec = items.iter().map(|(_, _, outs)| outs.len() as u32).collect(); let fks = self.append_stems_one_wave( items.len(), est_out, @@ -1527,6 +1527,7 @@ impl TxTable { }, |i, buf| encode_inwit_with_secret(&items[i].1, buf, Some(&self.secret)), |i, buf| encode_spent_zeros(items[i].2.len() as u32, buf), + &n_outs, )?; let ids: Vec<[u8; 32]> = items.iter().map(|(tx, _, _)| tx.txid).collect(); self.txids.append_batch(base, &ids)?; @@ -1591,6 +1592,10 @@ impl TxTable { encode_spent_slot_v17(0, fk)?; } } + let n_outs: Vec = items + .iter() + .map(|(pin, _)| pin.as_ref().1.len() as u32) + .collect(); let fks = self.append_stems_one_wave( items.len(), est_out, @@ -1608,6 +1613,7 @@ impl TxTable { encode_spent_slots(outs.len() as u32, pairs, buf) .expect("spent overlay prechecked"); }, + &n_outs, )?; let ids: Vec<[u8; 32]> = items.iter().map(|(pin, _)| pin.0.txid).collect(); self.txids.append_batch(base, &ids)?; @@ -1635,6 +1641,7 @@ impl TxTable { encode_out: impl FnMut(usize, &mut Vec), encode_in: impl FnMut(usize, &mut Vec), encode_sp: impl FnMut(usize, &mut Vec), + n_outs: &[u32], ) -> Result, StoreError> { let Some(p_out) = self.body.prepare_batch_encode(n, est_out, encode_out)? else { return Ok(Vec::new()); @@ -1660,7 +1667,7 @@ impl TxTable { "Class A append fk mismatch across stems", )); } - self.spent_off.note_starts(sp_base, &sp_starts); + self.spent_off.note_starts(sp_base, &sp_starts, n_outs); Ok(fks) } diff --git a/crates/rbitcoin-store/src/tx_table/spent_off.rs b/crates/rbitcoin-store/src/tx_table/spent_off.rs index d54355c7..80f5f7dd 100644 --- a/crates/rbitcoin-store/src/tx_table/spent_off.rs +++ b/crates/rbitcoin-store/src/tx_table/spent_off.rs @@ -1,4 +1,7 @@ -//! Sparse `spent.body` starts: prefix of `spent_record_len(n_out)` from txout. +//! Sparse `spent.body` starts: prefix of `spent_record_len(n_out)`. +//! +//! `n_out` is Class A append RAM, or LAYOUT17 meta loaded at open into that vec. +//! Spent-range APIs never read `txout.body`. use super::packed::spent_record_len; use super::TxRecord; @@ -9,13 +12,18 @@ use rbitcoin_primitives::{Fk, TableKind}; use std::path::{Path, PathBuf}; use std::sync::Mutex; -/// Checkpoint every N creates (fk 1, 1025, …). Between them, sum `n_out` from txout meta. +/// Checkpoint every N creates (fk 1, 1025, …). Between them, sum RAM `n_out`. const SPENT_OFF_STRIDE: u64 = 1024; const META_PEEK: u64 = 32; +struct SpentOffMem { + ckpts: Vec, + n_outs: Vec, +} + pub(super) struct SpentOff { path: PathBuf, - ckpts: Mutex>, + mem: Mutex, } pub(crate) fn unlink_leftover_spent_idx(dir: &Path) -> Result { @@ -57,11 +65,31 @@ pub(crate) fn unlink_leftover_spent_idx(dir: &Path) -> Result Ok(dropped) } +fn range_from_n_outs(n_outs: &[u32], base_fk: u64, id: u64, mut off: u64) -> Option<(u64, u64)> { + if id == 0 || (n_outs.len() as u64) < id { + return None; + } + let mut cur = base_fk; + while cur <= id { + let n_out = n_outs[(cur - 1) as usize]; + let len = spent_record_len(n_out); + if cur == id { + return Some((off, len)); + } + off = off.saturating_add(len); + cur += 1; + } + None +} + impl SpentOff { pub(super) fn new(dir: &Path) -> Self { Self { path: dir.join("spent.off"), - ckpts: Mutex::new(Vec::new()), + mem: Mutex::new(SpentOffMem { + ckpts: Vec::new(), + n_outs: Vec::new(), + }), } } @@ -84,57 +112,79 @@ impl SpentOff { for c in buf.chunks_exact(8) { ckpts.push(u64::from_le_bytes(c.try_into().unwrap())); } - *me.ckpts.lock().unwrap_or_else(|e| e.into_inner()) = ckpts; + me.mem.lock().unwrap_or_else(|e| e.into_inner()).ckpts = ckpts; Ok(me) } - pub(super) fn note_starts(&self, base_count: u64, starts: &[u64]) { + pub(super) fn note_starts(&self, base_count: u64, starts: &[u64], n_outs: &[u32]) { if starts.is_empty() { return; } - let mut g = self.ckpts.lock().unwrap_or_else(|e| e.into_inner()); + let mut g = self.mem.lock().unwrap_or_else(|e| e.into_inner()); for (i, &start) in starts.iter().enumerate() { let fk = base_count.saturating_add(1).saturating_add(i as u64); if !fk.saturating_sub(1).is_multiple_of(SPENT_OFF_STRIDE) { continue; } let idx = ((fk - 1) / SPENT_OFF_STRIDE) as usize; - if g.len() <= idx { - g.resize(idx + 1, 0); + if g.ckpts.len() <= idx { + g.ckpts.resize(idx + 1, 0); } - g[idx] = start; + g.ckpts[idx] = start; + } + if n_outs.len() == starts.len() && g.n_outs.len() as u64 == base_count { + g.n_outs.extend_from_slice(n_outs); } } pub(super) fn truncate_to_count(&self, new_count: u64) { - let mut g = self.ckpts.lock().unwrap_or_else(|e| e.into_inner()); + let mut g = self.mem.lock().unwrap_or_else(|e| e.into_inner()); if new_count == 0 { - g.clear(); + g.ckpts.clear(); + g.n_outs.clear(); return; } let keep = 1 + (new_count - 1) / SPENT_OFF_STRIDE; - g.truncate(keep as usize); + g.ckpts.truncate(keep as usize); + g.n_outs.truncate(new_count as usize); } + /// Open: durable `spent.off` plus RAM `n_out` from LAYOUT17 meta. pub(super) fn ensure_covering(&self, body: &VarTable, count: u64) -> Result<(), StoreError> { if count == 0 { - self.ckpts.lock().unwrap_or_else(|e| e.into_inner()).clear(); + let mut g = self.mem.lock().unwrap_or_else(|e| e.into_inner()); + g.ckpts.clear(); + g.n_outs.clear(); return Ok(()); } let need = 1 + (count - 1) / SPENT_OFF_STRIDE; - { - let g = self.ckpts.lock().unwrap_or_else(|e| e.into_inner()); - if g.len() as u64 >= need && g.first().copied() == Some(FILE_HEADER_LEN as u64) { - return Ok(()); - } + let (ckpts_ok, n_out_ok) = { + let g = self.mem.lock().unwrap_or_else(|e| e.into_inner()); + ( + g.ckpts.len() as u64 >= need + && g.ckpts.first().copied() == Some(FILE_HEADER_LEN as u64), + g.n_outs.len() as u64 == count, + ) + }; + if ckpts_ok && n_out_ok { + return Ok(()); + } + let (rebuilt, n_outs) = load_n_outs(body, count)?; + let mut g = self.mem.lock().unwrap_or_else(|e| e.into_inner()); + if !ckpts_ok { + g.ckpts = rebuilt; } - let rebuilt = rebuild_from_txout(body, count)?; - *self.ckpts.lock().unwrap_or_else(|e| e.into_inner()) = rebuilt; + g.n_outs = n_outs; Ok(()) } pub(super) fn flush(&self) -> Result<(), StoreError> { - let ckpts = self.ckpts.lock().unwrap_or_else(|e| e.into_inner()).clone(); + let ckpts = self + .mem + .lock() + .unwrap_or_else(|e| e.into_inner()) + .ckpts + .clone(); let f = if self.path.exists() { TableFile::open(&self.path, TableKind::ArrayLink)? } else { @@ -158,8 +208,8 @@ impl SpentOff { return Err(StoreError::InvalidFk); } let i = (fk - 1) / SPENT_OFF_STRIDE; - let g = self.ckpts.lock().unwrap_or_else(|e| e.into_inner()); - let Some(&off) = g.get(i as usize) else { + let g = self.mem.lock().unwrap_or_else(|e| e.into_inner()); + let Some(&off) = g.ckpts.get(i as usize) else { return Err(StoreError::Corrupt( "invariant: spent off checkpoint missing", )); @@ -170,12 +220,7 @@ impl SpentOff { Ok((i * SPENT_OFF_STRIDE + 1, off)) } - pub(super) fn range_for( - &self, - body: &VarTable, - fk: Fk, - count: u64, - ) -> Result<(u64, u64), StoreError> { + pub(super) fn range_for(&self, fk: Fk, count: u64) -> Result<(u64, u64), StoreError> { let id = fk.get().ok_or(StoreError::InvalidFk)?; if id == 0 { return Err(StoreError::InvalidFk); @@ -183,29 +228,17 @@ impl SpentOff { if id > count { return Err(StoreError::NotFound); } - let (base_fk, mut off) = self.base_for(id)?; + let (base_fk, off) = self.base_for(id)?; if base_fk > id { return Err(StoreError::Corrupt("invariant: spent off base")); } - let last_sum = id; // inclusive n_out through this fk - let n_outs = peek_output_counts(body, base_fk, last_sum)?; - if n_outs.len() as u64 != last_sum.saturating_sub(base_fk).saturating_add(1) { - return Err(StoreError::Corrupt("invariant: spent n_out walk")); - } - for (i, n_out) in n_outs.iter().enumerate() { - let cur = base_fk + i as u64; - let len = spent_record_len(*n_out); - if cur == id { - return Ok((off, len)); - } - off = off.saturating_add(len); - } - Err(StoreError::Corrupt("invariant: spent range walk")) + let g = self.mem.lock().unwrap_or_else(|e| e.into_inner()); + range_from_n_outs(&g.n_outs, base_fk, id, off) + .ok_or(StoreError::Corrupt("invariant: spent n_out missing")) } pub(super) fn ranges_batch( &self, - body: &VarTable, fks: &[Fk], count: u64, ) -> Result>, StoreError> { @@ -237,7 +270,13 @@ impl SpentOff { let lo = jobs[w].1; let hi = jobs[e - 1].1; let (base_fk, mut off) = self.base_for(lo)?; - let n_outs = peek_output_counts(body, base_fk, hi)?; + let n_outs = { + let g = self.mem.lock().unwrap_or_else(|e| e.into_inner()); + if (g.n_outs.len() as u64) < hi { + return Err(StoreError::Corrupt("invariant: spent n_out missing")); + } + g.n_outs[(base_fk - 1) as usize..(hi as usize)].to_vec() + }; let mut want = w; for (i, n_out) in n_outs.iter().enumerate() { let cur = base_fk + i as u64; @@ -256,16 +295,37 @@ impl SpentOff { Ok(out) } - pub(super) fn end_for(&self, body: &VarTable, count: u64) -> Result { + pub(super) fn end_for(&self, count: u64) -> Result { if count == 0 { return Ok(FILE_HEADER_LEN as u64); } - let (off, len) = self.range_for(body, Fk(count), count)?; + let (off, len) = self.range_for(Fk(count), count)?; Ok(off.saturating_add(len)) } } -fn peek_output_counts(body: &VarTable, first: u64, last: u64) -> Result, StoreError> { +fn load_n_outs(body: &VarTable, count: u64) -> Result<(Vec, Vec), StoreError> { + let mut ckpts = Vec::new(); + let mut all_n_outs = Vec::with_capacity(count as usize); + let mut off = FILE_HEADER_LEN as u64; + let mut first = 1u64; + while first <= count { + let last = first.saturating_add(SPENT_OFF_STRIDE - 1).min(count); + let n_outs = read_output_counts(body, first, last)?; + for (i, n_out) in n_outs.iter().enumerate() { + let fk = first + i as u64; + if (fk - 1).is_multiple_of(SPENT_OFF_STRIDE) { + ckpts.push(off); + } + off = off.saturating_add(spent_record_len(*n_out)); + } + all_n_outs.extend_from_slice(&n_outs); + first = last.saturating_add(1); + } + Ok((ckpts, all_n_outs)) +} + +fn read_output_counts(body: &VarTable, first: u64, last: u64) -> Result, StoreError> { if first == 0 || last < first { return Err(StoreError::InvalidFk); } @@ -293,22 +353,3 @@ fn peek_output_counts(body: &VarTable, first: u64, last: u64) -> Result } Ok(out) } - -fn rebuild_from_txout(body: &VarTable, count: u64) -> Result, StoreError> { - let mut ckpts = Vec::new(); - let mut off = FILE_HEADER_LEN as u64; - let mut first = 1u64; - while first <= count { - let last = first.saturating_add(SPENT_OFF_STRIDE - 1).min(count); - let n_outs = peek_output_counts(body, first, last)?; - for (i, n_out) in n_outs.iter().enumerate() { - let fk = first + i as u64; - if (fk - 1).is_multiple_of(SPENT_OFF_STRIDE) { - ckpts.push(off); - } - off = off.saturating_add(spent_record_len(*n_out)); - } - first = last.saturating_add(1); - } - Ok(ckpts) -} diff --git a/crates/rbitcoin-store/src/tx_table/tests.rs b/crates/rbitcoin-store/src/tx_table/tests.rs index b1b6600b..6cc48795 100644 --- a/crates/rbitcoin-store/src/tx_table/tests.rs +++ b/crates/rbitcoin-store/src/tx_table/tests.rs @@ -3639,6 +3639,29 @@ fn class_a_append_does_not_write_spent_idx() { let _ = std::fs::remove_dir_all(&dir); } +#[test] +fn spent_range_uses_append_n_out_not_txout_body() { + let dir = tempfile_dir("spent-ram-nout"); + let t = create_tiny(&dir); + let f0 = put_n_out(&t, 1, 0); + let f1 = put_n_out(&t, 2, 1); + let f3 = put_n_out(&t, 3, 3); + let want = t.spent_range_batch(&[f3, f0, f1]).unwrap(); + assert_eq!(want[0].map(|(_, l)| l), Some(spent_record_len(3))); + assert_eq!(want[1].map(|(_, l)| l), Some(spent_record_len(0))); + assert_eq!(want[2].map(|(_, l)| l), Some(spent_record_len(1))); + { + let body = dir.join("txout.body"); + let f = std::fs::OpenOptions::new().write(true).open(&body).unwrap(); + f.set_len(crate::file::FILE_HEADER_LEN as u64).unwrap(); + } + let got = t + .spent_range_batch(&[f3, f0, f1]) + .expect("spent range uses RAM n_out from append"); + assert_eq!(got, want); + let _ = std::fs::remove_dir_all(&dir); +} + #[test] fn open_unlinks_leftover_spent_idx() { let dir = tempfile_dir("spent-idx-leftover"); diff --git a/docs/invariants.md b/docs/invariants.md index f1eaf6da..c67a6beb 100644 --- a/docs/invariants.md +++ b/docs/invariants.md @@ -72,7 +72,7 @@ not head/idx. | Stage | Allowed IO | Forbidden | |-------|------------|-----------| -| **lookup** | `tx.head`, `txout.idx` (fk + ranges), `txid.body`, headers; leftover spent_range peeks txout **meta** (`n_out` only) | **`txout` outs / `inwit` decode** | +| **lookup** | `tx.head`, `txout.idx` (fk + ranges), `txid.body`, headers; spent_range from RAM `n_out` (append / open hydrate) | **`txout` outs / `inwit` decode**; **spent_range must not read `txout.body`** | | **load** | **`txout.body` outs by range** (from lookup stamp) | head, idx (`txout`), `txid.body`, `inwit` | | **scripts** | none | any store IO |