From 48d3b87724d9f1755d0125a279b264a221406e28 Mon Sep 17 00:00:00 2001 From: TAMARA LIPOWSKI Date: Wed, 29 Jul 2026 09:59:46 -0400 Subject: [PATCH 01/12] feat: require exclusive routes to beat public by 1 bps Exclusive-access routes are quoted only when their output net of gas clears the public reference by a configurable margin, EXCLUSIVE_ROUTE_IMPROVEMENT_BPS, floored at one atomic unit. The margin is added to the commitment, so it goes to the user and surplus capture starts above it. A route that clears the margin exactly is quoted with zero surplus. --- fynd-core/src/worker_pool_router/mod.rs | 103 ++++++++++++++++-------- 1 file changed, 71 insertions(+), 32 deletions(-) diff --git a/fynd-core/src/worker_pool_router/mod.rs b/fynd-core/src/worker_pool_router/mod.rs index 1250215b..ee5953bb 100644 --- a/fynd-core/src/worker_pool_router/mod.rs +++ b/fynd-core/src/worker_pool_router/mod.rs @@ -47,6 +47,14 @@ use crate::{ SolveParams, SurplusInfo, }; +/// Minimum improvement over the public route, in basis points of the public route's output net of +/// gas, that an exclusive-access route must deliver before it is quoted. +/// +/// The margin goes to the user: the commitment is raised by it, so the protocol only captures what +/// the route produces on top. A candidate that beats the public route by exactly this margin is +/// still quoted, with zero surplus. +const EXCLUSIVE_ROUTE_IMPROVEMENT_BPS: u64 = 1; + /// Which liquidity a solver pool (a group of workers) routes through, and therefore what its /// candidates mean in a quote. /// @@ -637,22 +645,27 @@ impl WorkerPoolRouter { /// holds the candidates from `ExclusiveAccess`-scoped pools (routes that may use exclusive /// components). /// +/// The user's net target is `required_net = public_net + margin`, where the margin is +/// `EXCLUSIVE_ROUTE_IMPROVEMENT_BPS` of `public_net`. A candidate whose net output falls below +/// `required_net` is skipped, so quoting an exclusive route always leaves the user better off than +/// the public market by at least that margin. +/// /// The committed amount is the larger of two lower bounds: -/// `max(public_amount_out, public_net + exclusive_gas)`. The first guarantees the quoted +/// `max(public_amount_out, required_net + exclusive_gas)`. The first guarantees the quoted /// `amount_out` is never below the public market's; the second guarantees the user — who pays -/// the exclusive route's gas — nets at least what the public route would leave them. +/// the exclusive route's gas — nets at least `required_net`. /// /// Which bound is larger depends on the gas comparison: -/// - `exclusive_gas > public_gas`: committed = `public_net + exclusive_gas`, which exceeds +/// - `exclusive_gas > public_gas`: committed = `required_net + exclusive_gas`, which exceeds /// `public_amount_out`. The user receives more tokens than the public quote and nets exactly -/// `public_net`. +/// `required_net`. /// - `exclusive_gas <= public_gas`: committed = `public_amount_out`. The user nets /// `public_amount_out − exclusive_gas`, which exceeds `public_net` by the gas difference. A -/// commitment of `public_net + exclusive_gas` would still leave the user whole and capture more, -/// but it is below `public_amount_out` — quoting less than the public market is ruled out, so the -/// gas difference stays with the user. +/// commitment of `required_net + exclusive_gas` would still leave the user whole and capture +/// more, but it is below `public_amount_out` — quoting less than the public market is ruled out, +/// so the gas difference stays with the user. /// -/// If the best exclusive-access candidate beats the public reference net-of-gas and produces at +/// If the best exclusive-access candidate clears `required_net` and produces at /// least the committed amount, this returns a new list whose head is the pinned surplus quote, /// followed by every public candidate as price-guard fallbacks. The pinned quote is the winning /// candidate with: @@ -661,7 +674,8 @@ impl WorkerPoolRouter { /// - an order-level [`SurplusInfo`] attached (observability). /// /// Otherwise `public_ranked` is returned unchanged. Either way the user is never worse off than -/// the public market — neither in quoted `amount_out` nor net of gas. +/// the public market — neither in quoted `amount_out` nor net of gas. A candidate that clears +/// `required_net` exactly is quoted with zero surplus: the whole improvement goes to the user. /// /// Per-leg attribution: the route's excess over the committed amount (`realized − committed`) /// is deducted from the exclusive legs — each leg absorbs what it can, capped at its own output, @@ -712,8 +726,9 @@ fn combine_with_surplus( return public_ranked; }; - // The candidate route must beat the committed reference net-of-gas. - if exclusive_candidate.amount_out_net_gas() <= committed.amount_out_net_gas() { + // The candidate route must beat the public reference net-of-gas by the improvement margin. + let required_net_amount_out = with_improvement_margin(committed.amount_out_net_gas()); + if exclusive_candidate.amount_out_net_gas() < &required_net_amount_out { return public_ranked; } @@ -734,10 +749,11 @@ fn combine_with_surplus( // The commitment is the larger of two lower bounds: the quoted amount_out is never below // the public market's, and the user — who pays the exclusive route's gas — never nets less - // than the public route would leave them (public_net + gas). Together with the strict net - // check above, these gates guarantee the route covers the committed amount. + // than the public route would leave them plus the improvement margin (required_net + gas). + // Together with the net check above, these gates guarantee the route covers the committed + // amount. let committed_amount_out = - (committed.amount_out_net_gas() + &exclusive_gas_cost).max(public_amount_out.clone()); + (&required_net_amount_out + &exclusive_gas_cost).max(public_amount_out.clone()); // What the hooks capture: everything the route produces above the commitment. let surplus_amount = exclusive_route_amount_out - &committed_amount_out; @@ -818,6 +834,17 @@ fn combine_with_surplus( result } +/// Returns `amount` raised by `EXCLUSIVE_ROUTE_IMPROVEMENT_BPS`. +/// +/// The margin is rounded up and is at least one atomic unit, so it never rounds away on small +/// amounts: an exclusive route that clears it always leaves the user strictly better off. +fn with_improvement_margin(amount: &BigUint) -> BigUint { + let denominator = BigUint::from(10_000u64); + let margin = (amount * BigUint::from(EXCLUSIVE_ROUTE_IMPROVEMENT_BPS) + (&denominator - 1u64)) / + &denominator; + amount + margin.max(BigUint::from(1u64)) +} + /// Returns `true` only for routes carrying exactly one exclusive leg, positioned as the terminal /// leg of its path. /// @@ -1835,19 +1862,31 @@ mod tests { /// Head selection across the gate case matrix. Each route is given as `(gross, net)`; /// expected is `(head amount_out, captured surplus)`. A surplus win prepends the pinned /// quote to the public fallbacks; otherwise the public ranking is returned unchanged. + /// + /// The improvement margin on these amounts is 1 (the one-atomic-unit floor) except in the + /// 1_000_000 cases, where 1 bps is 100. #[rstest] - #[case::exclusive_beats_public((900, 900), (950, 950), (900, 900, Some(50)))] + #[case::exclusive_beats_public((900, 900), (950, 950), (901, 901, Some(49)))] #[case::exclusive_below_public((950, 950), (900, 900), (950, 950, None))] #[case::exclusive_ties_public_net((900, 900), (900, 900), (900, 900, None))] - #[case::exclusive_marginally_better((999, 999), (1000, 1000), (999, 999, Some(1)))] + #[case::exclusive_marginally_better((999, 999), (1000, 1000), (1000, 1000, Some(0)))] #[case::exclusive_cannot_cover_public_gross((1000, 950), (990, 980), (1000, 950, None))] - // Gas-heavier exclusive route: the committed amount rises to max(1000, 950 + 140) = 1090 so - // the user still nets the public 950; the protocol captures 1100 - 1090 = 10. - #[case::exclusive_with_higher_gas((1000, 950), (1100, 960), (1090, 950, Some(10)))] + // Gas-heavier exclusive route: the committed amount rises to max(1000, 951 + 140) = 1091 so + // the user nets the public 950 plus the margin; the protocol captures 1100 - 1091 = 9. + #[case::exclusive_with_higher_gas((1000, 950), (1100, 960), (1091, 951, Some(9)))] // Gas-cheaper exclusive route: the public amount is the larger bound (committed = 1000); // the user keeps the // gas saving (nets 960) and the protocol captures 100. #[case::exclusive_with_lower_gas((1000, 950), (1100, 1060), (1000, 960, Some(100)))] + // 1 bps of 1_000_000 is 100: a candidate only 50 better than the public route is rejected. + #[case::exclusive_below_margin((1_000_000, 1_000_000), (1_000_050, 1_000_050), + (1_000_000, 1_000_000, None))] + // Exactly at the margin: the route is quoted and the whole 100 goes to the user, surplus 0. + #[case::exclusive_at_margin((1_000_000, 1_000_000), (1_000_100, 1_000_100), + (1_000_100, 1_000_100, Some(0)))] + // Above the margin: the user gets the 100, the protocol captures the remaining 400. + #[case::exclusive_above_margin((1_000_000, 1_000_000), (1_000_500, 1_000_500), + (1_000_100, 1_000_100, Some(400)))] fn test_combine_head_selection( #[case] public: (u64, u64), #[case] exclusive: (u64, u64), @@ -1956,14 +1995,14 @@ mod tests { .expect("should have an exclusive swap"); // committed_leg = leg.amount_out * committed_route_out / realized_route_out - // = 1000 * 900 / 1000 = 900 - assert_eq!(perm_swap.committed_amount_out(), Some(&BigUint::from(900u64)),); + // = 1000 * 901 / 1000 = 901 + assert_eq!(perm_swap.committed_amount_out(), Some(&BigUint::from(901u64)),); } #[test] fn test_combine_committed_leg_deduction() { - // leg = 995, committed = 900, realized = 1000: the route's excess (100) is deducted from - // the exclusive leg in full — committed_leg = 995 − 100 = 895, exactly, no rounding. + // leg = 995, committed = 901, realized = 1000: the route's excess (99) is deducted from + // the exclusive leg in full — committed_leg = 995 − 99 = 896, exactly, no rounding. let responses = OrderResponses { order_id: "test-order".to_string(), quotes: vec![ @@ -2002,7 +2041,7 @@ mod tests { .iter() .find(|s| policy.is_exclusive(s.protocol_component())) .expect("should have an exclusive swap"); - assert_eq!(perm_swap.committed_amount_out(), Some(&BigUint::from(895u64))); + assert_eq!(perm_swap.committed_amount_out(), Some(&BigUint::from(896u64))); } #[test] @@ -2025,9 +2064,9 @@ mod tests { #[test] fn test_combine_split_route_attribution() { - // Split route: public branch 600 + exclusive branch 500 = 1100 realized vs 1000 - // committed (zero gas). Only the exclusive leg is stamped; the public branch flows to - // the user untouched. + // Split route: public branch 600 + exclusive branch 500 = 1100 realized vs 1001 + // committed (zero gas, 1 unit improvement margin). Only the exclusive leg is stamped; + // the public branch flows to the user untouched. let responses = OrderResponses { order_id: "test-order".to_string(), quotes: vec![ @@ -2058,7 +2097,7 @@ mod tests { Some(&policy), ); - assert_eq!(*combined[0].amount_out(), BigUint::from(1000u64)); + assert_eq!(*combined[0].amount_out(), BigUint::from(1001u64)); let route = combined[0] .route() .expect("surplus quote should have a route"); @@ -2070,15 +2109,15 @@ mod tests { assert_eq!(public_leg.committed_amount_out(), None); // The public branch (600) pays out in full, so the entire excess - // (1100 − 1000 = 100) is deducted from the exclusive leg: committed_leg = 500 − 100 = - // 400. The user receives 600 + 400 = 1000 (exactly the committed amount) and the hook - // captures all 100. + // (1100 − 1001 = 99) is deducted from the exclusive leg: committed_leg = 500 − 99 = + // 401. The user receives 600 + 401 = 1001 (exactly the committed amount) and the hook + // captures all 99. let exclusive_leg = route .swaps() .iter() .find(|s| policy.is_exclusive(s.protocol_component())) .expect("should have an exclusive swap"); - assert_eq!(exclusive_leg.committed_amount_out(), Some(&BigUint::from(400u64))); + assert_eq!(exclusive_leg.committed_amount_out(), Some(&BigUint::from(401u64))); } /// Builds an `OrderResponses` where both quotes carry explicit `amount_out_net_gas`. From f70f8b55747f1b8da8c5ed4b35e2866910ef1345 Mon Sep 17 00:00:00 2001 From: TAMARA LIPOWSKI Date: Tue, 28 Jul 2026 12:24:02 -0400 Subject: [PATCH 02/12] feat(fynd-core): expose the market-event sender for test harnesses A harness that writes synthetic components into MarketData must also announce them, or workers never add the edge to their local graph. Gated behind the experimental feature, alongside BlockStepController. --- fynd-core/src/solver.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/fynd-core/src/solver.rs b/fynd-core/src/solver.rs index 623cdc7a..8542a58c 100644 --- a/fynd-core/src/solver.rs +++ b/fynd-core/src/solver.rs @@ -1167,6 +1167,16 @@ impl Solver { self.market_event_tx.subscribe() } + /// Returns a clone of the [`MarketEvent`] broadcast sender. + /// + /// A harness that writes synthetic components into [`MarketData`] must also announce them, or + /// workers never add the edge to their local graph. This is the same channel the Tycho feed + /// publishes real block updates on, so an injected event is indistinguishable from a real one. + #[cfg(feature = "experimental")] + pub fn market_event_sender(&self) -> broadcast::Sender { + self.market_event_tx.clone() + } + /// Submits a [`QuoteRequest`] to the worker pools and returns the best [`Quote`]. /// /// # Errors From 6adad491ed28941f77873ea76489b0cf9a0ea081 Mon Sep 17 00:00:00 2001 From: TAMARA LIPOWSKI Date: Tue, 28 Jul 2026 12:24:09 -0400 Subject: [PATCH 03/12] feat(hindsight): mock PropAMM pool to size underbid opportunity Mirrors the best real pool for a token pair onto a synthetic exclusive component at a configurable fee-free price, hidden from the public worker pools and visible to an exclusive-access twin of each. Each re-solved order then reports whether the PropAMM route won and how much fee it could have charged on top and still won. --- Cargo.lock | 2 + tools/hindsight/Cargo.toml | 5 + tools/hindsight/src/main.rs | 1 + tools/hindsight/src/propamm/mirror.rs | 388 ++++++++++++++++++++ tools/hindsight/src/propamm/mod.rs | 483 +++++++++++++++++++++++++ tools/hindsight/src/propamm/report.rs | 253 +++++++++++++ tools/hindsight/src/resolve/monitor.rs | 410 ++++++++++++++++++--- 7 files changed, 1483 insertions(+), 59 deletions(-) create mode 100644 tools/hindsight/src/propamm/mirror.rs create mode 100644 tools/hindsight/src/propamm/mod.rs create mode 100644 tools/hindsight/src/propamm/report.rs diff --git a/Cargo.lock b/Cargo.lock index 77ed0451..5fd8e5be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3835,6 +3835,7 @@ dependencies = [ "alloy", "anyhow", "async-trait", + "chrono", "clap", "futures 0.3.32", "fynd-client", @@ -3852,6 +3853,7 @@ dependencies = [ "tracing", "tracing-subscriber 0.3.23", "tycho-simulation", + "typetag", ] [[package]] diff --git a/tools/hindsight/Cargo.toml b/tools/hindsight/Cargo.toml index e74f6c1f..b80b542d 100644 --- a/tools/hindsight/Cargo.toml +++ b/tools/hindsight/Cargo.toml @@ -50,10 +50,15 @@ anyhow = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } +# Timestamps on the mock PropAMM component +chrono = { workspace = true } + # Serialization serde = { workspace = true } serde_json = { workspace = true } toml = { workspace = true } +# Registers the mock PropAMM pool state as a `Box` implementation +typetag = { workspace = true } [lints.clippy] pedantic = { level = "warn", priority = -1 } diff --git a/tools/hindsight/src/main.rs b/tools/hindsight/src/main.rs index 6047fa64..ceeb6e35 100644 --- a/tools/hindsight/src/main.rs +++ b/tools/hindsight/src/main.rs @@ -1,4 +1,5 @@ mod decoder; +mod propamm; mod report; mod resolve; mod telemetry; diff --git a/tools/hindsight/src/propamm/mirror.rs b/tools/hindsight/src/propamm/mirror.rs new file mode 100644 index 00000000..914807c9 --- /dev/null +++ b/tools/hindsight/src/propamm/mirror.rs @@ -0,0 +1,388 @@ +//! A [`ProtocolSim`] that mirrors another pool's curve at a configurable, fee-free price. +//! +//! The `PropAMM` pool that will go live is an Ekubo V3 pool whose base fee is 0 and whose per-swap +//! fee Fynd signs. This type stands in for the fee-free half of that: it mirrors a real pool's live +//! curve and scales the price by [`MirrorPool::from_price_pct`]'s percentage. It charges no fee, so +//! whatever the router later finds above the public commitment is exactly the fee headroom the +//! signed extension could charge and still win the trade. +//! +//! Wrapping rather than patching the mirrored state keeps this protocol-agnostic. Every concrete +//! state (`UniswapV3State`, `EkuboV3State`, a `vm:` pool) stores its fee somewhere different, so +//! rewriting a fee field would mean per-protocol surgery that breaks whenever +//! `tycho-simulation` changes shape. Delegating and scaling the result works for all of them. +//! +//! `query_pool_swap` is deliberately *not* implemented, so the trait default's +//! `"query_pool_swap not implemented"` error makes `PoolDepthComputation` fall back to its Brent +//! solver, which goes through [`ProtocolSim::get_amount_out`] and therefore sees the scaled price. +//! An implementation that delegated would report the mirrored pool's unscaled depth. + +use num_bigint::BigUint; +use serde::{Deserialize, Serialize}; +use tycho_simulation::{ + tycho_common::{models::token::Token, Bytes}, + tycho_core::simulation::{ + errors::{SimulationError, TransitionError}, + protocol_sim::{Balances, GetAmountOutResult, ProtocolSim}, + }, +}; + +/// Denominator of the price scale: the mirrored price is expressed in parts per million, so a +/// percentage resolves to 0.01 bps. `u32` throughout, which converts to `f64` losslessly. +const PRICE_SCALE: u32 = 1_000_000; + +/// Mirrors `inner`'s curve at `price_ppm` parts per million of its price, charging no fee. +/// +/// `price_ppm == PRICE_SCALE` mirrors the source exactly — the control case. Above it the mock +/// quotes better than the best real pool, below it worse. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct MirrorPool { + /// The mirrored pool's live state. Typetag-serialized, so any concrete state works. + inner: Box, + /// The mock's fee-free price as parts per million of the mirrored pool's price. + price_ppm: u32, +} + +impl MirrorPool { + /// Wraps `inner`, quoting at `price_pct` percent of its price. + /// + /// A percentage outside `[0, 400]` is clamped: the mock is a plausibility probe, and a price + /// hundreds of times the market's would only produce routes no real pool could ever fill. + pub(crate) fn from_price_pct(inner: Box, price_pct: f64) -> Self { + Self { inner, price_ppm: price_pct_to_ppm(price_pct) } + } + + /// The mock's fee-free price as a fraction of the mirrored pool's price. + pub(crate) fn price_factor(&self) -> f64 { + f64::from(self.price_ppm) / f64::from(PRICE_SCALE) + } + + /// Scales an output amount by the configured price, rounding down. + fn scale(&self, amount: &BigUint) -> BigUint { + amount * BigUint::from(self.price_ppm) / BigUint::from(PRICE_SCALE) + } +} + +/// Converts a percentage of the mirrored price into parts per million, clamped to a sane range. +/// +/// A non-finite input becomes `PRICE_SCALE` (mirror exactly), which is the safe default: it makes +/// the mock unable to win rather than able to win by an arbitrary amount. +// Truncation and sign loss are impossible after the clamp: the value is finite and in [0, 4e6]. +#[expect(clippy::cast_possible_truncation, clippy::cast_sign_loss)] +fn price_pct_to_ppm(price_pct: f64) -> u32 { + if !price_pct.is_finite() { + return PRICE_SCALE; + } + let ppm = (price_pct / 100.0 * f64::from(PRICE_SCALE)).clamp(0.0, 4.0 * f64::from(PRICE_SCALE)); + ppm.round() as u32 +} + +#[typetag::serde] +impl ProtocolSim for MirrorPool { + /// Zero: the mock is the fee-free curve, matching the live pool's base fee of 0. The per-swap + /// fee is the headroom the router discovers afterwards, not something priced in here. + /// + /// Reporting zero also sidesteps `ProtocolSim::fee`'s documented panic on protocols with + /// asymmetric fees (Uniswap V4, Rocketpool), which delegating to `inner` would inherit. + fn fee(&self) -> f64 { + 0.0 + } + + /// The mirrored pool's spot price, scaled. `spot_price` is quote-per-base — a cost — so a + /// better price is a smaller number, hence the division. + fn spot_price(&self, base: &Token, quote: &Token) -> Result { + let price = self.inner.spot_price(base, quote)?; + let factor = self.price_factor(); + if factor <= 0.0 { + return Err(SimulationError::FatalError( + "mirrored price factor is zero; the mock pool cannot quote".to_string(), + )); + } + Ok(price / factor) + } + + fn get_amount_out( + &self, + amount_in: BigUint, + token_in: &Token, + token_out: &Token, + ) -> Result { + let result = self + .inner + .get_amount_out(amount_in, token_in, token_out)?; + Ok(GetAmountOutResult { + amount: self.scale(&result.amount), + gas: result.gas, + new_state: Box::new(Self { inner: result.new_state, price_ppm: self.price_ppm }), + }) + } + + /// The mirrored pool's limits, with the output bound scaled. The input bound is unchanged: the + /// price scale is a price, not extra liquidity. + fn get_limits( + &self, + sell_token: Bytes, + buy_token: Bytes, + ) -> Result<(BigUint, BigUint), SimulationError> { + let (max_in, max_out) = self + .inner + .get_limits(sell_token, buy_token)?; + Ok((max_in, self.scale(&max_out))) + } + + fn delta_transition( + &mut self, + delta: tycho_simulation::tycho_core::dto::ProtocolStateDelta, + tokens: &std::collections::HashMap, + balances: &Balances, + ) -> Result<(), TransitionError> { + self.inner + .delta_transition(delta, tokens, balances) + } + + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } + + fn as_any_mut(&mut self) -> &mut dyn std::any::Any { + self + } + + fn eq(&self, other: &dyn ProtocolSim) -> bool { + other + .as_any() + .downcast_ref::() + .is_some_and(|other| { + self.price_ppm == other.price_ppm && self.inner.eq(other.inner.as_ref()) + }) + } +} + +#[cfg(test)] +mod tests { + use tycho_simulation::tycho_common::models::Chain; + + use super::*; + + /// A constant-price pool: one unit in, `rate` units out, so the price scale is the only thing + /// that moves the output. + #[derive(Debug, Clone, Serialize, Deserialize)] + struct FlatPool { + rate: u32, + } + + #[typetag::serde] + impl ProtocolSim for FlatPool { + fn fee(&self) -> f64 { + 0.003 + } + + fn spot_price(&self, _base: &Token, _quote: &Token) -> Result { + Ok(f64::from(self.rate)) + } + + fn get_amount_out( + &self, + amount_in: BigUint, + _token_in: &Token, + _token_out: &Token, + ) -> Result { + Ok(GetAmountOutResult { + amount: amount_in * BigUint::from(self.rate), + gas: BigUint::from(100_000u64), + new_state: Box::new(self.clone()), + }) + } + + fn get_limits( + &self, + _sell_token: Bytes, + _buy_token: Bytes, + ) -> Result<(BigUint, BigUint), SimulationError> { + Ok((BigUint::from(1_000u64), BigUint::from(1_000u64 * u64::from(self.rate)))) + } + + fn delta_transition( + &mut self, + _delta: tycho_simulation::tycho_core::dto::ProtocolStateDelta, + _tokens: &std::collections::HashMap, + _balances: &Balances, + ) -> Result<(), TransitionError> { + Ok(()) + } + + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } + + fn as_any_mut(&mut self) -> &mut dyn std::any::Any { + self + } + + fn eq(&self, other: &dyn ProtocolSim) -> bool { + other + .as_any() + .downcast_ref::() + .is_some_and(|other| self.rate == other.rate) + } + } + + fn token(symbol: &str) -> Token { + Token { + address: Bytes::from(vec![0x11; 20]), + symbol: symbol.to_string(), + decimals: 18, + tax: 0, + gas: vec![], + chain: Chain::Ethereum, + quality: 100, + } + } + + fn mirror(price_pct: f64) -> MirrorPool { + MirrorPool::from_price_pct(Box::new(FlatPool { rate: 1_000 }), price_pct) + } + + #[test] + fn test_get_amount_out_scales_by_the_configured_price() { + // (price_pct, expected output for 1_000 in at rate 1_000) + for (price_pct, expected) in [ + (100.0, 1_000_000u64), + (100.01, 1_000_100), + (100.05, 1_000_500), + (100.3, 1_003_000), + (99.9, 999_000), + ] { + let out = mirror(price_pct) + .get_amount_out(BigUint::from(1_000u64), &token("A"), &token("B")) + .expect("flat pool always quotes") + .amount; + assert_eq!(out, BigUint::from(expected), "price_pct = {price_pct}"); + } + } + + #[test] + fn test_hundred_percent_mirrors_the_source_exactly() { + // The control case the harness relies on: at 100% the mock cannot strictly beat its source, + // so the router must not select it over an equally-priced public route. + let source = FlatPool { rate: 1_000 }; + let amount = BigUint::from(7_777u64); + let mirrored = mirror(100.0) + .get_amount_out(amount.clone(), &token("A"), &token("B")) + .expect("mirror quotes") + .amount; + let direct = source + .get_amount_out(amount, &token("A"), &token("B")) + .expect("source quotes") + .amount; + assert_eq!(mirrored, direct); + } + + #[test] + fn test_fee_is_zero_so_headroom_is_measured_not_assumed() { + // The mock is the fee-free curve; the fee it could charge is what the router finds above + // the public commitment, not a number baked in here. + assert!(mirror(100.5).fee().abs() < f64::EPSILON); + } + + #[test] + fn test_new_state_keeps_the_price_scale() { + // Split routes chain swaps through `new_state`; if the scale were dropped there, only the + // first leg would be repriced. + let post_swap = mirror(100.5) + .get_amount_out(BigUint::from(1_000u64), &token("A"), &token("B")) + .expect("mirror quotes") + .new_state; + let out = post_swap + .get_amount_out(BigUint::from(1_000u64), &token("A"), &token("B")) + .expect("post-swap state quotes") + .amount; + assert_eq!(out, BigUint::from(1_005_000u64)); + } + + #[test] + fn test_spot_price_improves_with_the_price_scale() { + // spot_price is quote-per-base, so a better price is a lower number. + let base = mirror(100.0) + .spot_price(&token("A"), &token("B")) + .expect("price"); + let scaled = mirror(101.0) + .spot_price(&token("A"), &token("B")) + .expect("price"); + assert!(scaled < base, "{scaled} should undercut {base}"); + assert!((scaled - base / 1.01).abs() < 1e-9); + } + + #[test] + fn test_spot_price_errs_at_a_zero_price() { + // A zero price would divide by zero. Erroring keeps the pool out of routing instead of + // producing an infinite spot price that poisons every edge weight derived from it. + assert!(mirror(0.0) + .spot_price(&token("A"), &token("B")) + .is_err()); + } + + #[test] + fn test_get_limits_scales_output_bound_only() { + let (max_in, max_out) = mirror(101.0) + .get_limits(Bytes::from(vec![0x11; 20]), Bytes::from(vec![0x22; 20])) + .expect("limits"); + assert_eq!(max_in, BigUint::from(1_000u64)); + assert_eq!(max_out, BigUint::from(1_010_000u64)); + } + + #[test] + fn test_price_pct_clamps_and_defaults_safely() { + assert_eq!(price_pct_to_ppm(100.0), PRICE_SCALE); + assert_eq!(price_pct_to_ppm(-5.0), 0, "a negative price floors at zero"); + assert_eq!(price_pct_to_ppm(10_000.0), 4 * PRICE_SCALE, "an absurd price is capped"); + // NaN must mirror exactly rather than win by an arbitrary amount. + assert_eq!(price_pct_to_ppm(f64::NAN), PRICE_SCALE); + assert_eq!(price_pct_to_ppm(f64::INFINITY), PRICE_SCALE); + } + + #[test] + fn test_query_pool_swap_reports_unimplemented() { + // PoolDepthComputation matches this exact message to fall back to its Brent solver, which + // reads amounts through get_amount_out and therefore sees the price scale. Delegating here + // would instead report the mirrored pool's unscaled depth. + use tycho_simulation::tycho_core::simulation::protocol_sim::{ + Price, QueryPoolSwapParams, SwapConstraint, + }; + + let params = QueryPoolSwapParams::new( + token("A"), + token("B"), + SwapConstraint::TradeLimitPrice { + limit: Price::new(BigUint::from(1u64), BigUint::from(1u64)), + tolerance: 0.0, + min_amount_in: None, + max_amount_in: None, + }, + ); + + let result = mirror(100.5).query_pool_swap(¶ms); + assert!( + matches!(&result, Err(SimulationError::FatalError(msg)) + if msg == "query_pool_swap not implemented"), + "expected the trait default's fatal error" + ); + } + + #[test] + fn test_gas_is_the_mirrored_pools_gas() { + // The PropAMM hop costs roughly what its source hop costs; inventing a gas number would + // bias amount_out_net_gas and therefore the win/loss comparison. + let result = mirror(105.0) + .get_amount_out(BigUint::from(1u64), &token("A"), &token("B")) + .expect("mirror quotes"); + assert_eq!(result.gas, BigUint::from(100_000u64)); + } +} diff --git a/tools/hindsight/src/propamm/mod.rs b/tools/hindsight/src/propamm/mod.rs new file mode 100644 index 00000000..00a4957a --- /dev/null +++ b/tools/hindsight/src/propamm/mod.rs @@ -0,0 +1,483 @@ +//! Mock `PropAMM` pool for measuring what dynamic underbidding would win, before the pool exists. +//! +//! # What this measures +//! +//! The live `PropAMM` will be an Ekubo V3 pool at base fee 0 whose per-swap fee Fynd signs just low +//! enough to underbid the best competing route. Two independent quantities decide whether that +//! works, so the harness keeps them separate: +//! +//! 1. **The pool's fee-free price**, set by `--propamm-price-pct` as a percentage of the best real +//! pool's price for the pair. `100` means "the `PropAMM` holds exactly the best price we can +//! see"; `100.05` means it holds a price 5 bps better. This is the input — the assumption about +//! how the pool is positioned. +//! 2. **The fee it could charge and still win**, which the harness *measures*. The mock quotes at +//! the configured price with no fee at all, so whatever the router finds above the public +//! commitment is exactly the headroom the signed extension could take. That is reported per +//! trade and averaged over the run as `fee_headroom_bps`. +//! +//! So a run answers: "with the pool at N% of the market's best price, it would have won this share +//! of flow, and could have charged this much fee on top and still won it." +//! +//! Mechanically the harness inserts a synthetic component into the running solver's market state, +//! mirrors the best real pool for the pair onto it at the configured price (see +//! [`mirror::MirrorPool`]), and lets Fynd's existing exclusive-access routing do the rest: public +//! worker pools never see the mock, their exclusive-access twins do, and the router reports the +//! surplus whenever the mock route beats the public reference. +//! +//! # Why the market state, not the Tycho stream +//! +//! Rewriting each `Update` before it reaches `TychoFeed` would need a mirrored state cached across +//! blocks, because a pool only appears in `Update::states` on blocks where it changed. Writing +//! straight into [`MarketData`] avoids that entirely: `MarketState` holds the latest state for +//! every component regardless of which block last touched it, so every block has a source to +//! mirror. +//! +//! Two things the injection still has to get right: +//! +//! - **Announce the component, once.** Workers build their graph from `MarketEvent::MarketUpdated`, +//! so a component written into `MarketState` without an event is invisible to routing. The first +//! injection announces it as added; later ones as updated, which is also what drives incremental +//! recomputation of its spot price and depth. +//! - **Wait for derived data.** Edge weights come from spot prices. Solving in the gap between the +//! injected event and the recomputation it triggers would rank the mock on a stale weight, so +//! [`Injector::inject`] blocks until the mock's spot price is recomputed at the target block. +//! +//! # Not for production +//! +//! Scaffolding for ENG-6157. The mock prices a pool that does not exist on chain, so any calldata +//! it produces is unexecutable. + +mod mirror; +pub(crate) mod report; + +use std::{ + collections::HashMap, + str::FromStr, + time::{Duration, Instant}, +}; + +use chrono::NaiveDateTime; +use fynd_core::{feed::market_data::MarketData, MarketEvent, Solver}; +use mirror::MirrorPool; +use num_bigint::BigUint; +use tokio::sync::broadcast; +use tracing::{debug, info, warn}; +use tycho_simulation::{ + tycho_common::{ + models::{protocol::ProtocolComponent, token::Token, Address, Chain, ChangeType}, + Bytes, + }, + tycho_core::simulation::protocol_sim::ProtocolSim, +}; + +/// Component id of the mock pool. Address-shaped so anything that treats a component id as a pool +/// address stays well-formed, and recognisable in logs and JSONL. +pub(crate) const MOCK_COMPONENT_ID: &str = "0x9797979797979797979797979797979797979797"; + +/// Protocol system the mock reports. `ekubo_v3` is what the live pool will be, and it selects the +/// Ekubo swap encoder — the one path that knows how to carry a signed exclusive swap. +const MOCK_PROTOCOL_SYSTEM: &str = "ekubo_v3"; + +/// Placeholder for Ekubo's `SignedExclusiveSwap` extension, which is not deployed yet. Only the +/// encoder and the EIP-712 domain read it, so a placeholder produces a well-formed — but +/// deliberately unexecutable — payload. +const SIGNED_EXCLUSIVE_SWAP_ADDRESS: &str = "0x5519ed5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e"; + +/// How long to wait for the mock's spot price to be recomputed at the target block before solving +/// anyway. Generous: it competes with a whole block's incremental computation, not just the mock's. +const DERIVED_DATA_WAIT: Duration = Duration::from_secs(30); +/// Poll interval while waiting for that recomputation. +const DERIVED_DATA_POLL: Duration = Duration::from_millis(25); + +/// Which pool to mirror, and at what fee-free price. +#[derive(Debug, Clone)] +pub(crate) struct MirrorConfig { + /// First token of the mirrored pair. + pub token_a: Address, + /// Second token of the mirrored pair. + pub token_b: Address, + /// The mock's fee-free price as a percentage of the best real pool's price for the pair. + /// `100.0` mirrors that pool exactly and must never win — the harness's control case. + pub price_pct: f64, + /// `token_a` amount, in whole units, used to rank candidate source pools each block. Picking + /// by realized output at a representative size, rather than by TVL, keeps the mirror on + /// the pool that actually prices best for the sizes being re-solved. + pub probe_units: f64, + /// Chain the mock component reports. + pub chain: Chain, +} + +impl MirrorConfig { + /// Parses a `--propamm-pair` value of two comma-separated token addresses. + pub(crate) fn parse_pair(pair: &[String]) -> anyhow::Result<(Address, Address)> { + let [token_a, token_b] = pair else { + anyhow::bail!("--propamm-pair needs exactly two token addresses, got {}", pair.len()); + }; + Ok(( + Bytes::from_str(token_a) + .map_err(|e| anyhow::anyhow!("invalid token address {token_a}: {e}"))?, + Bytes::from_str(token_b) + .map_err(|e| anyhow::anyhow!("invalid token address {token_b}: {e}"))?, + )) + } +} + +/// What one injection did, for logging. +#[derive(Debug)] +pub(crate) struct Injected { + /// Component id of the pool that was mirrored this block. + pub source_component: String, + /// The best real pool's price: `token_b` per whole unit of `token_a`, at the probe size. + /// Watching this across blocks is how you tell a live mirror from one stuck on a stale state. + pub source_price: f64, + /// The mock's fee-free price, i.e. `source_price` scaled by `--propamm-price-pct`. + pub mock_price: f64, + /// Whether the mock's spot price was recomputed before the wait expired. + pub derived_data_ready: bool, +} + +/// Writes the mock pool into a running solver's market state, once per block. +pub(crate) struct Injector { + config: MirrorConfig, + events: broadcast::Sender, + /// Candidate source pools for the pair, resolved on first use. Pools for a pair change far + /// more slowly than blocks do, so this is refreshed only while still empty. + candidates: Vec, + /// Whether the mock has been announced as an added component. Announcing twice would re-add an + /// existing graph edge. + announced: bool, +} + +impl Injector { + /// Creates an injector publishing on the solver's market-event channel. + pub(crate) fn new(solver: &Solver, config: MirrorConfig) -> Self { + Self { + config, + events: solver.market_event_sender(), + candidates: Vec::new(), + announced: false, + } + } + + /// Mirrors the best source pool onto the mock component for `block`. + /// + /// Returns `Ok(None)` when no pool for the configured pair carries state yet — early blocks of + /// a fresh feed, or a pair that simply is not indexed. + pub(crate) async fn inject( + &mut self, + solver: &Solver, + block: u64, + ) -> anyhow::Result> { + let market = solver.market_data(); + + if self.candidates.is_empty() { + self.candidates = find_candidates(&market, &self.config).await; + if self.candidates.is_empty() { + return Ok(None); + } + info!( + candidates = self.candidates.len(), + "resolved candidate source pools for the mirrored pair" + ); + } + + let Some((source_component, source_state, mirrored_price)) = + best_source(&market, &self.candidates, &self.config).await + else { + return Ok(None); + }; + + let mock = MirrorPool::from_price_pct(source_state, self.config.price_pct); + let price_factor = mock.price_factor(); + let mirrored: Box = Box::new(mock); + let first_injection = !self.announced; + { + let mut state = market.write().await; + if first_injection { + state.upsert_components([mock_component(&self.config)]); + } + state.update_states([(MOCK_COMPONENT_ID.to_string(), mirrored)]); + } + + let event = if first_injection { + self.announced = true; + MarketEvent::MarketUpdated { + added_components: HashMap::from([( + MOCK_COMPONENT_ID.to_string(), + vec![self.config.token_a.clone(), self.config.token_b.clone()], + )]), + removed_components: Vec::new(), + updated_components: Vec::new(), + } + } else { + MarketEvent::MarketUpdated { + added_components: HashMap::new(), + removed_components: Vec::new(), + updated_components: vec![MOCK_COMPONENT_ID.to_string()], + } + }; + self.events + .send(event) + .map_err(|e| anyhow::anyhow!("no market-event receivers left: {e}"))?; + + let derived_data_ready = wait_for_derived_data(solver, block).await; + if !derived_data_ready { + warn!( + block, + "mock pool's spot price was not recomputed within {}s; its edge weight may be \ + stale for this block", + DERIVED_DATA_WAIT.as_secs() + ); + } + + let mock_price = mirrored_price * price_factor; + debug!( + block, + source_component, + source_price = mirrored_price, + mock_price, + price_pct = self.config.price_pct, + "mirrored source pool onto the mock PropAMM component" + ); + Ok(Some(Injected { + source_component, + source_price: mirrored_price, + mock_price, + derived_data_ready, + })) + } +} + +/// Component ids holding both tokens of the configured pair, excluding the mock itself. +/// +/// Multi-token pools (Curve, Balancer) qualify: the mirror only ever quotes the configured pair, so +/// what matters is that the source can price it. +async fn find_candidates(market: &MarketData, config: &MirrorConfig) -> Vec { + let state = market.read().await; + state + .component_topology() + .into_iter() + .filter(|(id, tokens)| { + id != MOCK_COMPONENT_ID && + tokens.contains(&config.token_a) && + tokens.contains(&config.token_b) + }) + .map(|(id, _)| id) + .collect() +} + +/// The candidate quoting the most `token_b` for the probe amount of `token_a`. +/// +/// Returns its id, a clone of its state, and the price it quoted (`token_b` per whole unit of +/// `token_a`). Candidates that cannot price the pair — no state yet, or a simulation error — are +/// skipped rather than failing the block: a single broken pool must not stop the run. +async fn best_source( + market: &MarketData, + candidates: &[String], + config: &MirrorConfig, +) -> Option<(String, Box, f64)> { + let state = market.read().await; + let token_a = state + .get_token(&config.token_a)? + .clone(); + let token_b = state + .get_token(&config.token_b)? + .clone(); + + let probe_amount = to_atomic(config.probe_units, &token_a); + let mut best: Option<(String, Box, BigUint)> = None; + for id in candidates { + let Some(sim) = state.get_simulation_state(id) else { + continue; + }; + let Ok(quoted) = sim.get_amount_out(probe_amount.clone(), &token_a, &token_b) else { + continue; + }; + if best + .as_ref() + .is_none_or(|(_, _, best_out)| quoted.amount > *best_out) + { + best = Some((id.clone(), sim.clone_box(), quoted.amount)); + } + } + + let (id, sim, probe_out) = best?; + let price = if config.probe_units > 0.0 { + to_units(&probe_out, &token_b) / config.probe_units + } else { + 0.0 + }; + Some((id, sim, price)) +} + +/// Blocks until the mock's spot price has been recomputed at `block`, or the wait expires. +/// +/// Returns whether it landed in time. Both conditions matter: the entry proves the mock entered the +/// computation at all, and the block proves the entry is this block's, not the previous one's. +async fn wait_for_derived_data(solver: &Solver, block: u64) -> bool { + let started = Instant::now(); + let derived = solver.derived_data(); + loop { + { + let guard = derived.read().await; + let computed_at_block = guard.spot_prices_block() == Some(block); + let includes_mock = guard + .spot_prices() + .is_some_and(|prices| { + prices + .keys() + .any(|(component_id, _, _)| component_id == MOCK_COMPONENT_ID) + }); + if computed_at_block && includes_mock { + return true; + } + } + if started.elapsed() >= DERIVED_DATA_WAIT { + return false; + } + tokio::time::sleep(DERIVED_DATA_POLL).await; + } +} + +/// The mock's `ProtocolComponent`. +/// +/// The three static attributes are what `EkuboV3SwapEncoder` and the exclusive-swap signer read: +/// `extension` (20 bytes), `fee` (8-byte big-endian `u64`, zero because the `PropAMM`'s base fee is +/// zero and the per-swap fee arrives in the signature), and `pool_type_config` (4 bytes). +fn mock_component(config: &MirrorConfig) -> ProtocolComponent { + let extension = Bytes::from_str(SIGNED_EXCLUSIVE_SWAP_ADDRESS) + .expect("the extension placeholder is a valid address literal"); + ProtocolComponent { + id: MOCK_COMPONENT_ID.to_string(), + protocol_system: MOCK_PROTOCOL_SYSTEM.to_string(), + protocol_type_name: "swap".to_string(), + chain: config.chain, + tokens: vec![config.token_a.clone(), config.token_b.clone()], + static_attributes: HashMap::from([ + ("extension".to_string(), extension), + ("fee".to_string(), Bytes::from(0u64)), + ("pool_type_config".to_string(), Bytes::from(0u32)), + ]), + change: ChangeType::default(), + creation_tx: Bytes::default(), + created_at: NaiveDateTime::default(), + contract_addresses: Vec::new(), + } +} + +/// Whether a component is the mock `PropAMM` pool — the predicate the solver's +/// [`ExclusivityPolicy`](fynd_core) is built from. +pub(crate) fn is_mock_component(component: &ProtocolComponent) -> bool { + component.id == MOCK_COMPONENT_ID +} + +/// Formats a token amount in whole units, for logging. +pub(crate) fn to_units(amount: &BigUint, token: &Token) -> f64 { + report::biguint_to_f64(amount) / 10f64.powi(decimals_exponent(token)) +} + +/// Converts whole units into the token's atomic units. +/// +/// Returns zero for anything that cannot be represented — non-finite, sub-atomic, or beyond `u64`. +/// A zero probe makes every candidate quote zero, so the caller picks arbitrarily rather than +/// silently mirroring a pool chosen by a garbage number. +// Truncation and sign loss are impossible after the guard: the value is finite and in [1, 2^63). +// The bound is a power of two, so it is exactly representable and the comparison is not +// approximate. +#[expect(clippy::cast_possible_truncation, clippy::cast_sign_loss)] +fn to_atomic(units: f64, token: &Token) -> BigUint { + let atomic = units * 10f64.powi(decimals_exponent(token)); + if !atomic.is_finite() || atomic < 1.0 || atomic >= 2f64.powi(63) { + return BigUint::ZERO; + } + BigUint::from(atomic as u64) +} + +/// A token's decimals as the exponent both conversions use, defaulting to 18 on an absurd value. +fn decimals_exponent(token: &Token) -> i32 { + i32::try_from(token.decimals).unwrap_or(18) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_mock_component_carries_the_attributes_the_encoder_reads() { + let config = MirrorConfig { + token_a: Bytes::from(vec![0x11; 20]), + token_b: Bytes::from(vec![0x22; 20]), + price_pct: 100.05, + probe_units: 1.0, + chain: Chain::Ethereum, + }; + let component = mock_component(&config); + + assert_eq!(component.protocol_system, MOCK_PROTOCOL_SYSTEM); + assert_eq!(component.static_attributes["extension"].len(), 20); + assert_eq!(component.static_attributes["fee"].len(), 8); + assert_eq!(component.static_attributes["pool_type_config"].len(), 4); + assert_eq!(component.tokens, vec![config.token_a, config.token_b]); + } + + #[test] + fn test_mock_component_declares_zero_base_fee() { + // The live pool's base fee must be 0 (Ekubo reverts with PoolFeeMustBeZero otherwise); the + // whole fee is carried in the signed payload. + let config = MirrorConfig { + token_a: Bytes::from(vec![0x11; 20]), + token_b: Bytes::from(vec![0x22; 20]), + price_pct: 100.0, + probe_units: 1.0, + chain: Chain::Ethereum, + }; + assert_eq!(mock_component(&config).static_attributes["fee"], Bytes::from(0u64)); + } + + #[test] + fn test_is_mock_component_matches_only_the_mock() { + let config = MirrorConfig { + token_a: Bytes::from(vec![0x11; 20]), + token_b: Bytes::from(vec![0x22; 20]), + price_pct: 100.0, + probe_units: 1.0, + chain: Chain::Ethereum, + }; + let mut real = mock_component(&config); + real.id = "0xdeadbeef".to_string(); + + assert!(is_mock_component(&mock_component(&config))); + assert!(!is_mock_component(&real)); + } + + #[test] + fn test_parse_pair_accepts_two_addresses() { + let pair = vec![ + "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2".to_string(), + "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48".to_string(), + ]; + let (token_a, token_b) = MirrorConfig::parse_pair(&pair).expect("valid pair"); + assert_eq!(token_a.len(), 20); + assert_eq!(token_b.len(), 20); + } + + #[test] + fn test_parse_pair_rejects_wrong_arity() { + assert!(MirrorConfig::parse_pair(&["0x11".to_string()]).is_err()); + assert!(MirrorConfig::parse_pair(&[]).is_err()); + } + + #[test] + fn test_to_units_scales_by_decimals() { + let token = Token { + address: Bytes::from(vec![0x11; 20]), + symbol: "USDC".to_string(), + decimals: 6, + tax: 0, + gas: vec![], + chain: Chain::Ethereum, + quality: 100, + }; + assert!((to_units(&BigUint::from(2_500_000u64), &token) - 2.5).abs() < 1e-9); + } +} diff --git a/tools/hindsight/src/propamm/report.rs b/tools/hindsight/src/propamm/report.rs new file mode 100644 index 00000000..bdf929d6 --- /dev/null +++ b/tools/hindsight/src/propamm/report.rs @@ -0,0 +1,253 @@ +//! Collects what the mock `PropAMM` pool won, block by block. +//! +//! The mock quotes at its configured fee-free price and charges nothing, so when the router selects +//! it, everything the route produces above the public commitment is **fee headroom**: the fee the +//! signed extension could have charged and still won the trade. The router already computes exactly +//! that split — [`OrderQuote::committed_amount_out`] is what the user is promised and +//! [`OrderQuote::surplus_amount`] is the excess — so this module only has to read it, express it in +//! bps, and total it up. +//! +//! Because the quoted `amount_out` is pinned to the commitment, the mock pool never changes +//! hindsight's own win/loss verdict. It only adds this second, orthogonal measurement. + +use std::sync::Mutex; + +use alloy::primitives::Address; +use fynd_core::OrderQuote; +use num_bigint::BigUint; + +use crate::propamm::MOCK_COMPONENT_ID; + +/// One re-solved order's mock-`PropAMM` outcome. +#[derive(Debug, Clone)] +pub(crate) struct Observation { + /// Order input token. + pub token_in: Address, + /// Order output token. + pub token_out: Address, + /// Order size, in `token_in` atomic units. + pub amount_in: BigUint, + /// Whether the winning route ran through the mock `PropAMM` pool. + pub won: bool, + /// The public-market output the user is committed to. `None` when the mock pool didn't win. + pub committed_amount_out: Option, + /// Output the mock produced above the commitment — the fee it could have charged. `None` when + /// it didn't win. + pub fee_headroom: Option, +} + +impl Observation { + /// Reads a re-solved quote's mock-`PropAMM` outcome. + /// + /// A win is a winning route that contains the mock component — not merely a non-empty surplus, + /// which can legitimately round to zero on a marginal beat. + pub(crate) fn from_quote( + quote: &OrderQuote, + token_in: Address, + token_out: Address, + amount_in: BigUint, + ) -> Self { + let won = quote.route().is_some_and(|route| { + route + .swaps() + .iter() + .any(|swap| swap.component_id() == MOCK_COMPONENT_ID) + }); + Self { + token_in, + token_out, + amount_in, + won, + committed_amount_out: quote.committed_amount_out().cloned(), + fee_headroom: quote.surplus_amount().cloned(), + } + } + + /// The fee the mock could have charged, as a fraction of the committed output, in basis points. + /// + /// This is the headline per-trade number: "the pool could have taken this much fee and the user + /// would still have been better off than on the public market." `None` when the pool didn't win + /// or the commitment is zero. + pub(crate) fn fee_headroom_bps(&self) -> Option { + let committed = biguint_to_f64(self.committed_amount_out.as_ref()?); + let headroom = biguint_to_f64(self.fee_headroom.as_ref()?); + if committed <= 0.0 { + return None; + } + Some(headroom / committed * 10_000.0) + } +} + +/// Running totals over a whole monitor run. +#[derive(Debug, Default, Clone, Copy)] +pub(crate) struct Totals { + /// Orders re-solved into a successful quote. + pub solved: u64, + /// Of those, how many routed through the mock `PropAMM` pool. + pub won: u64, + /// Fee headroom valued in USD at top-of-block prices. + pub headroom_usd: f64, + /// Committed output valued in USD — the flow the pool captured. + pub captured_flow_usd: f64, +} + +impl Totals { + /// Share of solved orders the mock pool won, as a percentage. Zero when nothing solved. + // Precision loss is irrelevant: these are trade counts, far below f64's exact-integer range. + #[expect(clippy::cast_precision_loss)] + pub(crate) fn winrate_pct(&self) -> f64 { + if self.solved == 0 { + return 0.0; + } + self.won as f64 / self.solved as f64 * 100.0 + } + + /// Fee headroom as a fraction of captured flow, in basis points — the average fee the pool + /// could have charged across everything it won. Zero when it captured no flow. + pub(crate) fn avg_fee_headroom_bps(&self) -> f64 { + if self.captured_flow_usd <= 0.0 { + return 0.0; + } + self.headroom_usd / self.captured_flow_usd * 10_000.0 + } +} + +/// Shared sink the solve path writes observations into and the block loop drains. +/// +/// `StepAdapter::solve` takes `&self`, so the sink owns its mutability. A `Mutex` is right here: +/// contention is one lock per re-solved order. +#[derive(Debug, Default)] +pub(crate) struct Stats { + pending: Mutex>, + totals: Mutex, +} + +impl Stats { + /// Records one re-solved order. + pub(crate) fn record(&self, observation: Observation) { + if let Ok(mut pending) = self.pending.lock() { + pending.push(observation); + } + } + + /// Takes everything recorded since the last drain. + pub(crate) fn drain(&self) -> Vec { + self.pending + .lock() + .map(|mut pending| std::mem::take(&mut *pending)) + .unwrap_or_default() + } + + /// Folds a block's observations into the run totals and returns the updated snapshot. + pub(crate) fn accumulate( + &self, + observations: &[Observation], + headroom_usd: f64, + captured_flow_usd: f64, + ) -> Totals { + let Ok(mut totals) = self.totals.lock() else { + return Totals::default(); + }; + totals.solved += observations.len() as u64; + totals.won += observations + .iter() + .filter(|o| o.won) + .count() as u64; + totals.headroom_usd += headroom_usd; + totals.captured_flow_usd += captured_flow_usd; + *totals + } + + /// The run totals so far. + pub(crate) fn totals(&self) -> Totals { + self.totals + .lock() + .map(|totals| *totals) + .unwrap_or_default() + } +} + +/// Converts a `BigUint` to `f64` for ratio reporting. Saturates to infinity beyond `f64` range, +/// which the callers guard against by dividing only by positive finite values. +pub(crate) fn biguint_to_f64(value: &BigUint) -> f64 { + value + .to_string() + .parse::() + .unwrap_or(f64::INFINITY) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn observation(won: bool, committed: u64, headroom: u64) -> Observation { + Observation { + token_in: Address::from([0x11; 20]), + token_out: Address::from([0x22; 20]), + amount_in: BigUint::from(1_000u64), + won, + committed_amount_out: won.then(|| BigUint::from(committed)), + fee_headroom: won.then(|| BigUint::from(headroom)), + } + } + + #[test] + fn test_fee_headroom_bps_from_committed_and_headroom() { + let observed = observation(true, 1_000_000, 500); + assert!( + (observed + .fee_headroom_bps() + .expect("a win reports headroom") - + 5.0) + .abs() < + 1e-9 + ); + } + + #[test] + fn test_fee_headroom_bps_absent_when_pool_lost() { + assert!(observation(false, 0, 0) + .fee_headroom_bps() + .is_none()); + } + + #[test] + fn test_fee_headroom_bps_absent_on_zero_commitment() { + // A zero commitment would divide by zero; the ratio is undefined, not infinite. + let mut observed = observation(true, 0, 500); + observed.committed_amount_out = Some(BigUint::ZERO); + assert!(observed.fee_headroom_bps().is_none()); + } + + #[test] + fn test_accumulate_sums_across_blocks() { + let stats = Stats::default(); + stats.accumulate(&[observation(true, 100, 1), observation(false, 0, 0)], 2.0, 400.0); + let totals = stats.accumulate(&[observation(true, 100, 1)], 1.0, 200.0); + + assert_eq!(totals.solved, 3); + assert_eq!(totals.won, 2); + assert!((totals.headroom_usd - 3.0).abs() < 1e-9); + assert!((totals.captured_flow_usd - 600.0).abs() < 1e-9); + assert!((totals.winrate_pct() - 200.0 / 3.0).abs() < 1e-9); + // 3 USD of headroom on 600 USD of captured flow = 50 bps. + assert!((totals.avg_fee_headroom_bps() - 50.0).abs() < 1e-9); + } + + #[test] + fn test_empty_totals_report_zero_rather_than_nan() { + let totals = Totals::default(); + assert!(totals.winrate_pct().abs() < f64::EPSILON); + assert!(totals.avg_fee_headroom_bps().abs() < f64::EPSILON); + } + + #[test] + fn test_drain_empties_the_sink() { + let stats = Stats::default(); + stats.record(observation(true, 100, 1)); + stats.record(observation(false, 0, 0)); + + assert_eq!(stats.drain().len(), 2); + assert!(stats.drain().is_empty(), "a second drain must not repeat observations"); + } +} diff --git a/tools/hindsight/src/resolve/monitor.rs b/tools/hindsight/src/resolve/monitor.rs index b9ef2b92..08c12fd4 100644 --- a/tools/hindsight/src/resolve/monitor.rs +++ b/tools/hindsight/src/resolve/monitor.rs @@ -10,6 +10,7 @@ use std::{ future::Future, pin::Pin, + sync::Arc, time::{Duration, Instant}, }; @@ -23,7 +24,7 @@ use fynd_core::{ parse_chain, EncodingOptions, Order, OrderQuote, OrderSide, QuoteOptions, QuoteRequest, QuoteStatus, }, - BlockStepController, FyndBuilder, Solver, + BlockStepController, FyndBuilder, LiquidityScope, Solver, }; use num_bigint::BigUint; use tracing::{debug, info, warn}; @@ -31,7 +32,7 @@ use tycho_simulation::tycho_common::models::{Address as CoreAddress, Chain}; use crate::{ decoder::{DecodedTrade, Decoder, Registry}, - provider_from, + propamm, provider_from, resolve::{resolve_block_range, Outcome, SolvedAmount, SteppingSolver}, telemetry, usd::Prices, @@ -116,6 +117,44 @@ pub(crate) struct MonitorArgs { /// reason; filter downstream for the improvement or coverage view #[arg(long)] pub comparisons_dir: Option, + + /// Mirror a mock `PropAMM` pool onto this token pair, as two comma-separated addresses + /// (`--propamm-pair 0xWETH,0xUSDC`). The mock carries the best real pool's live curve at the + /// price set by `--propamm-price-pct` and charges no fee. It is hidden from the public worker + /// pools and visible to a parallel exclusive-access twin of each configured pool, so each + /// re-solved order reports whether the `PropAMM` route won and how much fee it could have + /// charged on top and still won. Off when omitted, and never usable against a real chain — the + /// mock has no pool behind it + #[arg(long, env = "PROPAMM_PAIR", value_delimiter = ',')] + pub propamm_pair: Option>, + + /// The mock pool's fee-free price, as a percentage of the best real pool's price for the pair. + /// `100` positions it exactly at the best price we can see (the control case — it cannot then + /// strictly beat the public market); `100.05` positions it 5 bps better. The fee it could + /// charge on top is measured, not set here + #[arg(long, env = "PROPAMM_PRICE_PCT", default_value_t = 100.0)] + pub propamm_price_pct: f64, + + /// Trade size, in whole units of the pair's first token, used each block to pick which real + /// pool to mirror. Set it near the sizes being re-solved so the mirror tracks the pool that + /// actually prices those trades best + #[arg(long, env = "PROPAMM_PROBE_UNITS", default_value_t = 1.0)] + pub propamm_probe_units: f64, + + /// Append one JSON line per re-solved order that the mock `PropAMM` won, with the committed + /// output, the fee headroom in bps, and both valued in USD + #[arg(long, env = "PROPAMM_JSONL")] + pub propamm_jsonl: Option, +} + +/// The mock-`PropAMM` scaffold attached to one monitor run. +/// +/// `stats` is created once per run so the totals span solver rebuilds; `injector` is rebuilt with +/// each solver, because it publishes on that solver's market-event channel. +struct PropAmmHarness { + injector: tokio::sync::Mutex, + stats: Arc, + jsonl: Option, } /// Drives the in-process solver, stepping the chain one block per `SteppingSolver::advance`. @@ -123,6 +162,8 @@ struct StepAdapter<'a> { solver: &'a Solver, controller: &'a BlockStepController, timeout_ms: u64, + /// Present when `--propamm-pair` is set; drives mock-pool injection and collects its outcomes. + propamm: Option<&'a PropAmmHarness>, } impl StepAdapter<'_> { @@ -135,51 +176,20 @@ impl StepAdapter<'_> { .last_updated() .map(fynd_core::BlockInfo::number) } -} -#[async_trait] -impl SteppingSolver for StepAdapter<'_> { - async fn solve(&self, token_in: Address, token_out: Address, amount_in: U256) -> Outcome { - let Ok(amount) = amount_in.to_string().parse::() else { - return Outcome::Unsolvable("unparseable amount_in".to_string()); - }; - // Placeholder receiver: routing/amounts are receiver-independent; it only fills the encoded - // calldata's recipient. Encoding is requested so each quote carries its on-chain - // transaction (note: this refines gas estimates and a failed encode yields - // Unsolvable). - let order = Order::new( - CoreAddress::from(token_in.into_array()), - CoreAddress::from(token_out.into_array()), - amount, - OrderSide::Sell, - CoreAddress::from([0x11u8; 20]), - ); - let request = QuoteRequest::new( - vec![order], - QuoteOptions::default() - .with_timeout_ms(self.timeout_ms) - .with_encoding_options(EncodingOptions::new(0.005)), - ); - - match self.solver.quote(request).await { - Ok(quote) => quote.orders().first().map_or_else( - || Outcome::Unsolvable("solver returned no order quote".to_string()), - order_quote_to_outcome, - ), - Err(e) => Outcome::Unsolvable(format!("solve error: {e}")), - } - } - - async fn advance(&self) -> anyhow::Result<()> { + /// Releases the next block and waits until the solver applies it. + /// + /// An error means the feed died — either its stream ended (peek returns None once the gating + /// task exits) or it jammed without ending (no block within `FEED_DEAD_TIMEOUT`). The caller + /// rebuilds the solver on any error. + async fn step_block(&self) -> anyhow::Result<()> { let before = self.current_block().await; self.controller .trigger_next_block() .map_err(|_| anyhow::anyhow!("tycho stream ended (trigger channel closed)"))?; // Deterministic barrier: wait until the solver applies a block strictly newer than - // `before`. An error here means the feed died — either its stream ended (peek returns - // None once the gating task exits) or it jammed without ending (no block within - // FEED_DEAD_TIMEOUT). The caller rebuilds the solver on any error. + // `before`. let stall_started = Instant::now(); let mut next_warn = stall_started + STALL_WARN_INTERVAL; loop { @@ -212,6 +222,98 @@ impl SteppingSolver for StepAdapter<'_> { } } } + + /// Re-mirrors the mock `PropAMM` pool onto the freshly applied block's state. + /// + /// A failure here leaves the mock holding the previous block's state, which skews its quotes + /// but does not invalidate the block's public comparison — so it warns and continues rather + /// than ending the session. + async fn mirror_propamm_pool(&self) { + let Some(harness) = self.propamm else { + return; + }; + let Some(block) = self.current_block().await else { + return; + }; + match harness + .injector + .lock() + .await + .inject(self.solver, block) + .await + { + Ok(Some(injected)) => debug!( + block, + source = injected.source_component, + source_price = injected.source_price, + mock_price = injected.mock_price, + derived_data_ready = injected.derived_data_ready, + "mirrored the mock PropAMM pool" + ), + Ok(None) => warn!( + block, + "no source pool for the mirrored pair carries state yet; the mock PropAMM is \ + inactive this block" + ), + Err(e) => warn!(block, "failed to mirror the mock PropAMM pool: {e}"), + } + } +} + +#[async_trait] +impl SteppingSolver for StepAdapter<'_> { + async fn solve(&self, token_in: Address, token_out: Address, amount_in: U256) -> Outcome { + let Ok(amount) = amount_in.to_string().parse::() else { + return Outcome::Unsolvable("unparseable amount_in".to_string()); + }; + // Placeholder receiver: routing/amounts are receiver-independent; it only fills the encoded + // calldata's recipient. Encoding is requested so each quote carries its on-chain + // transaction (note: this refines gas estimates and a failed encode yields + // Unsolvable). + let order = Order::new( + CoreAddress::from(token_in.into_array()), + CoreAddress::from(token_out.into_array()), + amount.clone(), + OrderSide::Sell, + CoreAddress::from([0x11u8; 20]), + ); + let request = QuoteRequest::new( + vec![order], + QuoteOptions::default() + .with_timeout_ms(self.timeout_ms) + .with_encoding_options(EncodingOptions::new(0.005)), + ); + + match self.solver.quote(request).await { + Ok(quote) => { + let Some(order_quote) = quote.orders().first() else { + return Outcome::Unsolvable("solver returned no order quote".to_string()); + }; + if let Some(harness) = self.propamm { + // Only successful quotes count, so the winrate's denominator is orders the + // PropAMM actually had a chance at. + if order_quote.status() == QuoteStatus::Success { + harness + .stats + .record(propamm::report::Observation::from_quote( + order_quote, + token_in, + token_out, + amount, + )); + } + } + order_quote_to_outcome(order_quote) + } + Err(e) => Outcome::Unsolvable(format!("solve error: {e}")), + } + } + + async fn advance(&self) -> anyhow::Result<()> { + self.step_block().await?; + self.mirror_propamm_pool().await; + Ok(()) + } } fn order_quote_to_outcome(quote: &OrderQuote) -> Outcome { @@ -332,6 +434,23 @@ async fn build_solver( .add_pool(name, pool) .map_err(|e| anyhow::anyhow!("failed to add worker pool {name}: {e}"))?; } + if cfg.propamm_pair.is_some() { + // Twin every configured pool with an exclusive-access copy: same algorithm and hop limits, + // so the two scopes differ only in whether they can see the mock pool. Anything less would + // confound the PropAMM's advantage with an algorithm difference. The unscoped originals + // stay public — `FyndBuilder` hands the exclusivity policy to every pool that does not + // opt into `LiquidityScope::All`. + builder = builder.exclusivity_policy(propamm::is_mock_component); + for (name, pool) in pools_config.pools() { + let twin = pool + .clone() + .with_liquidity_scope(LiquidityScope::All); + let twin_name = format!("{name}__propamm"); + builder = builder + .add_pool(&twin_name, &twin) + .map_err(|e| anyhow::anyhow!("failed to add worker pool {twin_name}: {e}"))?; + } + } builder .build_with_step_controller() .await @@ -429,23 +548,7 @@ pub(crate) async fn run(cfg: MonitorArgs) -> anyhow::Result<()> { .await .map_err(|e| anyhow::anyhow!("failed to resolve protocols: {e}"))?; - // Load worker pools like `fynd serve`: the default path falls back to the built-in default - // pools when absent; custom paths that don't exist fail fast. - let default_path = std::path::Path::new("worker_pools.toml"); - let pools_config = - if cfg.worker_pools_config.as_path() == default_path && !default_path.exists() { - info!("worker_pools.toml not found; using Fynd's built-in default pools"); - fynd_rpc::config::WorkerPoolsConfig::builtin_default() - } else { - fynd_rpc::config::WorkerPoolsConfig::load_from_file(&cfg.worker_pools_config).map_err( - |e| { - anyhow::anyhow!( - "failed to load worker pools config {}: {e}", - cfg.worker_pools_config.display() - ) - }, - )? - }; + let pools_config = load_pools_config(&cfg.worker_pools_config)?; let mut decoder = Decoder::new( provider_from(&cfg.chain.rpc_url)?, @@ -466,6 +569,10 @@ pub(crate) async fn run(cfg: MonitorArgs) -> anyhow::Result<()> { None => None, }; + // Parsed before the first (multi-minute) solver build, so a typo in the pair fails immediately. + let propamm_config = propamm_config(&cfg, chain)?; + let propamm_stats = Arc::new(propamm::report::Stats::default()); + let mut totals = Totals::default(); let pacing = Pacing::for_chain(chain, cfg.max_lag_blocks); info!( @@ -488,8 +595,21 @@ pub(crate) async fn run(cfg: MonitorArgs) -> anyhow::Result<()> { built = build_solver(&cfg, chain, &protocols, &pools_config) => built?, }; loop { - let adapter = - StepAdapter { solver: &solver, controller: &controller, timeout_ms: cfg.timeout_ms }; + // Rebuilt with each solver: the injector publishes on that solver's market-event channel. + // `propamm_stats` is shared, so the totals span rebuilds. + let propamm = propamm_config + .clone() + .map(|config| PropAmmHarness { + injector: tokio::sync::Mutex::new(propamm::Injector::new(&solver, config)), + stats: Arc::clone(&propamm_stats), + jsonl: cfg.propamm_jsonl.clone(), + }); + let adapter = StepAdapter { + solver: &solver, + controller: &controller, + timeout_ms: cfg.timeout_ms, + propamm: propamm.as_ref(), + }; let reason = tokio::select! { biased; () = &mut shutdown => { @@ -521,9 +641,171 @@ pub(crate) async fn run(cfg: MonitorArgs) -> anyhow::Result<()> { (solver, controller) = built; } solver.shutdown(); + if let Some(config) = propamm_config.as_ref() { + log_propamm_summary(config, &propamm_stats.totals()); + } Ok(()) } +/// Loads the worker pools like `fynd serve` does. +/// +/// The default path falls back to the built-in default pools when absent; a custom path that does +/// not exist fails fast, because an operator who named a file meant that file. +fn load_pools_config( + path: &std::path::Path, +) -> anyhow::Result { + let default_path = std::path::Path::new("worker_pools.toml"); + if path == default_path && !default_path.exists() { + info!("worker_pools.toml not found; using Fynd's built-in default pools"); + return Ok(fynd_rpc::config::WorkerPoolsConfig::builtin_default()); + } + fynd_rpc::config::WorkerPoolsConfig::load_from_file(path) + .map_err(|e| anyhow::anyhow!("failed to load worker pools config {}: {e}", path.display())) +} + +/// Builds the mock-`PropAMM` config from the CLI, or `None` when `--propamm-pair` is absent. +fn propamm_config( + cfg: &MonitorArgs, + chain: Chain, +) -> anyhow::Result> { + let Some(pair) = cfg.propamm_pair.as_deref() else { + return Ok(None); + }; + let (token_a, token_b) = propamm::MirrorConfig::parse_pair(pair)?; + let config = propamm::MirrorConfig { + token_a, + token_b, + price_pct: cfg.propamm_price_pct, + probe_units: cfg.propamm_probe_units, + chain, + }; + info!( + token_a = %config.token_a, + token_b = %config.token_b, + price_pct = config.price_pct, + probe_units = config.probe_units, + "mock PropAMM enabled; its quotes are not executable" + ); + Ok(Some(config)) +} + +/// Logs the whole run's mock-`PropAMM` result: the assumption that went in, and the winrate and fee +/// headroom that came out. +fn log_propamm_summary(config: &propamm::MirrorConfig, totals: &propamm::report::Totals) { + info!( + price_pct = config.price_pct, + solved_orders = totals.solved, + propamm_wins = totals.won, + winrate_pct = format!("{:.1}", totals.winrate_pct()), + captured_flow_usd = format!("{:.0}", totals.captured_flow_usd), + fee_headroom_usd = format!("{:.2}", totals.headroom_usd), + fee_headroom_bps = format!("{:.2}", totals.avg_fee_headroom_bps()), + "mock PropAMM run summary" + ); +} + +/// Values one block's mock-`PropAMM` observations in USD, folds them into the run totals, logs the +/// running picture, and appends the wins to JSONL. +/// +/// Amounts are valued at top-of-block prices, matching the headline improvement metric. Both halves +/// are valued: the committed output answers "how much flow the pool captured", the headroom answers +/// "how much fee it could have charged on that flow", and their ratio is that fee in bps. +fn report_propamm_block( + harness: &PropAmmHarness, + block: u64, + prices: &Prices, + observations: &[propamm::report::Observation], +) { + let mut headroom_usd = 0.0; + let mut captured_flow_usd = 0.0; + let mut lines = Vec::new(); + + for observed in observations.iter().filter(|o| o.won) { + let value_usd = |amount: Option<&BigUint>| { + amount + .and_then(biguint_to_u256_opt) + .and_then(|amount| prices.value_usd(observed.token_out, amount)) + }; + let headroom = value_usd(observed.fee_headroom.as_ref()); + let committed = value_usd(observed.committed_amount_out.as_ref()); + headroom_usd += headroom.unwrap_or(0.0); + captured_flow_usd += committed.unwrap_or(0.0); + + if harness.jsonl.is_some() { + lines.push(serde_json::json!({ + "block": block, + "token_in": observed.token_in.to_string(), + "token_out": observed.token_out.to_string(), + "amount_in": observed.amount_in.to_string(), + "committed_amount_out": observed + .committed_amount_out + .as_ref() + .map(std::string::ToString::to_string), + "fee_headroom": observed + .fee_headroom + .as_ref() + .map(std::string::ToString::to_string), + "fee_headroom_bps": observed.fee_headroom_bps(), + "committed_usd": committed, + "fee_headroom_usd": headroom, + })); + } + } + + let totals = harness + .stats + .accumulate(observations, headroom_usd, captured_flow_usd); + let block_wins = observations + .iter() + .filter(|o| o.won) + .count(); + info!( + block, + block_orders = observations.len(), + block_wins, + block_fee_headroom_usd = format!("{headroom_usd:.2}"), + run_winrate_pct = format!("{:.1}", totals.winrate_pct()), + run_fee_headroom_usd = format!("{:.2}", totals.headroom_usd), + run_fee_headroom_bps = format!("{:.2}", totals.avg_fee_headroom_bps()), + "mock PropAMM block result" + ); + + if let Some(path) = harness.jsonl.as_ref() { + if let Err(e) = append_jsonl(path, &lines) { + warn!(path = %path.display(), "failed to append PropAMM JSONL: {e}"); + } + } +} + +/// Appends one JSON object per line, creating the file if absent. +fn append_jsonl(path: &std::path::Path, lines: &[serde_json::Value]) -> std::io::Result<()> { + use std::io::Write as _; + + if lines.is_empty() { + return Ok(()); + } + let mut file = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(path)?; + for line in lines { + writeln!(file, "{line}")?; + } + file.flush() +} + +/// Converts a `BigUint` to `U256`, or `None` when it does not fit — the amount is then left +/// unvalued rather than silently reported as zero. +fn biguint_to_u256_opt(value: &BigUint) -> Option { + let bytes = value.to_bytes_be(); + if bytes.len() > 32 { + return None; + } + let mut buf = [0u8; 32]; + buf[32 - bytes.len()..].copy_from_slice(&bytes); + Some(U256::from_be_bytes(buf)) +} + /// Drive one solver session: step blocks and re-solve each block's settled trades until the run /// completes or the feed dies. async fn run_session( @@ -629,6 +911,12 @@ async fn run_session( if let Some(rotating) = comparisons.as_mut() { super::jsonl::write_comparisons(rotating.writer(), &ranges, &prices_top, &prices_back); } + // Drained unconditionally when the harness is on, so a block whose trades all failed to + // solve still clears the sink rather than carrying observations into the next block. + if let Some(harness) = adapter.propamm { + let observations = harness.stats.drain(); + report_propamm_block(harness, target, &prices_top, &observations); + } let elapsed_s = start.elapsed().as_secs_f64(); telemetry::record_block_seconds(elapsed_s); @@ -776,6 +1064,10 @@ mod tests { max_blocks: Some(1), max_lag_blocks: Some(100), comparisons_dir: None, + propamm_pair: None, + propamm_price_pct: 100.05, + propamm_probe_units: 1.0, + propamm_jsonl: None, }) .await .expect("monitor should process one block without error"); From 91f5d1f51d03fbcdb755a426728f617d63157792 Mon Sep 17 00:00:00 2001 From: TAMARA LIPOWSKI Date: Tue, 28 Jul 2026 12:52:34 -0400 Subject: [PATCH 04/12] feat(hindsight): report the mock PropAMM's winrate and fee headroom Splits the two quantities: --propamm-price-pct sets the pool's fee-free price as a percentage of the best real pool's, and the fee it could charge on top is measured from the router's surplus. Each outcome rides in the comparisons JSONL, so the offline report renders a section with the run winrate, captured flow, and a per-order-pair breakdown. --- tools/hindsight/CLAUDE.md | 36 +++- tools/hindsight/src/propamm/mod.rs | 51 +++-- tools/hindsight/src/propamm/report.rs | 114 ++++++++-- tools/hindsight/src/report/aggregate.rs | 263 ++++++++++++++++++++++++ tools/hindsight/src/report/html.rs | 126 +++++++++++- tools/hindsight/src/report/record.rs | 85 +++++++- tools/hindsight/src/resolve/jsonl.rs | 23 ++- tools/hindsight/src/resolve/monitor.rs | 204 +++++++++--------- 8 files changed, 762 insertions(+), 140 deletions(-) diff --git a/tools/hindsight/CLAUDE.md b/tools/hindsight/CLAUDE.md index 0a2d8246..0c946eaf 100644 --- a/tools/hindsight/CLAUDE.md +++ b/tools/hindsight/CLAUDE.md @@ -27,13 +27,16 @@ and takes neither. back-of-block (state N), and emits `RangeComparison` JSONL records. Exposes a Prometheus metrics endpoint (`--metrics-port`). `--max-lag-blocks` (default 100, ~20 min on mainnet) bounds how far it may fall behind chain head before rebuilding the solver. + `--propamm-pair ` additionally injects a mock PropAMM pool (see below). - **`report`** — Offline: read the `comparisons-YYYY-MM-DD.jsonl` files a `monitor` run wrote (`--comparisons-dir`) and render a single self-contained HTML file (`-o`, defaults to `/report.html`) with the dashboard's value views — the headline Fynd savings, win rate, and median savings bps; the verdict split by trade count and by volume; per-solver/venue breakdowns; top-saving trades; and the unsolved token tail. `--venue ` (repeatable, case-insensitive) - restricts the report to those venues. No chain, Tycho, or network access. + restricts the report to those venues. No chain, Tycho, or network access. Records from a + `--propamm-pair` run also get a "Mock PropAMM" section (winrate, captured flow, fee headroom, and + a per-order-pair breakdown). ## Environment @@ -43,6 +46,9 @@ and takes neither. | `ALLIUM_API_KEY` | Allium API key (`verify` only) | | `ALLIUM_QUERY_ID` | Saved Allium query ID (`verify` only) | | `HINDSIGHT_REGISTRY` | Override path for the decoder address-book TOML | +| `PROPAMM_PAIR` | Token pair the mock PropAMM mirrors, comma-separated (`monitor` only) | +| `PROPAMM_PRICE_PCT` | Mock PropAMM's fee-free price as a percentage of the best real pool's | +| `PROPAMM_PROBE_UNITS` | Trade size used to pick which real pool the mock mirrors | ## Architecture @@ -101,6 +107,34 @@ self-contained HTML file. | `aggregate.rs` | Pure aggregations over the records (verdicts, coverage, savings, per-group, movers) | | `html.rs` | Renders the aggregates to a self-contained HTML file (inline CSS, `
` bars, no assets) | +### Mock PropAMM (`src/propamm/`) + +Test scaffolding for ENG-6157 — sizes what a dynamic-underbidding PropAMM pool would win before the +pool exists. Off unless `monitor --propamm-pair` is set. + +| File | Purpose | +|---|---| +| `mod.rs` | `Injector` — writes a synthetic exclusive component into the running solver's `MarketState` once per block and announces it on the market-event channel | +| `mirror.rs` | `MirrorPool` — a `ProtocolSim` that delegates to the best real pool for the pair and scales its price by `--propamm-price-pct`, charging no fee | +| `report.rs` | Per-order outcomes and run totals; `Record` is what lands in the comparisons JSONL | + +Two knobs, deliberately separate: + +- **Input** — `--propamm-price-pct` is the pool's fee-free price as a percentage of the best real + pool's price for the pair. `100` places it exactly at the best price we can see (the control case: + the router requires a *strict* beat, so it must never win); `100.05` places it 5 bps better. +- **Output** — *fee headroom*: the mock charges nothing, so whatever the router finds above the + public commitment is the fee the signed extension could have charged and still won the trade. + Reported per trade, per order pair, and as a flow-weighted run average. + +Fynd's existing exclusive-access routing does the rest: `FyndBuilder::exclusivity_policy` hides the +mock from every configured worker pool, and each pool is twinned with a `liquidity_scope = "all"` +copy that sees it. Because the router pins a surplus quote's `amount_out` to the public commitment, +the mock never changes hindsight's own win/loss verdicts — it only adds this second measurement. + +Requires `fynd-core`'s `experimental` feature for `Solver::market_event_sender`. Not for production: +the mock prices a pool that does not exist on chain, so any calldata it produces is unexecutable. + ### Verdict model Each re-solved trade produces a `top` (optimistic, state N-1) and `back` (pessimistic, state N) diff --git a/tools/hindsight/src/propamm/mod.rs b/tools/hindsight/src/propamm/mod.rs index 00a4957a..0c7d1844 100644 --- a/tools/hindsight/src/propamm/mod.rs +++ b/tools/hindsight/src/propamm/mod.rs @@ -134,6 +134,8 @@ pub(crate) struct Injected { pub mock_price: f64, /// Whether the mock's spot price was recomputed before the wait expired. pub derived_data_ready: bool, + /// The mirrored pair as token symbols, e.g. `WETH/USDC`. + pub pair_label: String, } /// Writes the mock pool into a running solver's market state, once per block. @@ -181,13 +183,11 @@ impl Injector { ); } - let Some((source_component, source_state, mirrored_price)) = - best_source(&market, &self.candidates, &self.config).await - else { + let Some(source) = best_source(&market, &self.candidates, &self.config).await else { return Ok(None); }; - let mock = MirrorPool::from_price_pct(source_state, self.config.price_pct); + let mock = MirrorPool::from_price_pct(source.state, self.config.price_pct); let price_factor = mock.price_factor(); let mirrored: Box = Box::new(mock); let first_injection = !self.announced; @@ -230,20 +230,21 @@ impl Injector { ); } - let mock_price = mirrored_price * price_factor; + let mock_price = source.price * price_factor; debug!( block, - source_component, - source_price = mirrored_price, + source_component = source.component_id, + source_price = source.price, mock_price, price_pct = self.config.price_pct, "mirrored source pool onto the mock PropAMM component" ); Ok(Some(Injected { - source_component, - source_price: mirrored_price, + source_component: source.component_id, + source_price: source.price, mock_price, derived_data_ready, + pair_label: source.pair_label, })) } } @@ -266,16 +267,29 @@ async fn find_candidates(market: &MarketData, config: &MirrorConfig) -> Vec, + /// The price it quoted: `token_b` per whole unit of `token_a`. + price: f64, + /// The pair as token symbols, e.g. `WETH/USDC`. + pair_label: String, +} + +/// The candidate pool the mock mirrors this block. async fn best_source( market: &MarketData, candidates: &[String], config: &MirrorConfig, -) -> Option<(String, Box, f64)> { +) -> Option { let state = market.read().await; let token_a = state .get_token(&config.token_a)? @@ -301,13 +315,18 @@ async fn best_source( } } - let (id, sim, probe_out) = best?; + let (component_id, state, probe_out) = best?; let price = if config.probe_units > 0.0 { to_units(&probe_out, &token_b) / config.probe_units } else { 0.0 }; - Some((id, sim, price)) + Some(BestSource { + component_id, + state, + price, + pair_label: format!("{}/{}", token_a.symbol, token_b.symbol), + }) } /// Blocks until the mock's spot price has been recomputed at `block`, or the wait expires. diff --git a/tools/hindsight/src/propamm/report.rs b/tools/hindsight/src/propamm/report.rs index bdf929d6..66f5e429 100644 --- a/tools/hindsight/src/propamm/report.rs +++ b/tools/hindsight/src/propamm/report.rs @@ -15,18 +15,21 @@ use std::sync::Mutex; use alloy::primitives::Address; use fynd_core::OrderQuote; use num_bigint::BigUint; +use serde::Serialize; use crate::propamm::MOCK_COMPONENT_ID; /// One re-solved order's mock-`PropAMM` outcome. #[derive(Debug, Clone)] pub(crate) struct Observation { - /// Order input token. - pub token_in: Address, - /// Order output token. + /// Order output token — the token both amounts below are denominated in, and the one their USD + /// valuation is taken against. The order's input token and size are not repeated here: the + /// comparison record this joins onto already carries them. pub token_out: Address, - /// Order size, in `token_in` atomic units. - pub amount_in: BigUint, + /// Whether the solver produced a successful quote at all. Unsuccessful solves are still + /// recorded so the sink stays index-aligned with the block's trades, which is how each + /// observation is joined back to its comparison record. + pub solved: bool, /// Whether the winning route ran through the mock `PropAMM` pool. pub won: bool, /// The public-market output the user is committed to. `None` when the mock pool didn't win. @@ -41,22 +44,18 @@ impl Observation { /// /// A win is a winning route that contains the mock component — not merely a non-empty surplus, /// which can legitimately round to zero on a marginal beat. - pub(crate) fn from_quote( - quote: &OrderQuote, - token_in: Address, - token_out: Address, - amount_in: BigUint, - ) -> Self { - let won = quote.route().is_some_and(|route| { - route - .swaps() - .iter() - .any(|swap| swap.component_id() == MOCK_COMPONENT_ID) - }); + pub(crate) fn from_quote(quote: &OrderQuote, token_out: Address) -> Self { + let solved = quote.status() == fynd_core::types::QuoteStatus::Success; + let won = solved && + quote.route().is_some_and(|route| { + route + .swaps() + .iter() + .any(|swap| swap.component_id() == MOCK_COMPONENT_ID) + }); Self { - token_in, token_out, - amount_in, + solved, won, committed_amount_out: quote.committed_amount_out().cloned(), fee_headroom: quote.surplus_amount().cloned(), @@ -120,6 +119,9 @@ impl Totals { pub(crate) struct Stats { pending: Mutex>, totals: Mutex, + /// The mirrored pair as token symbols, e.g. `WETH/USDC`. Resolved once the feed has loaded the + /// tokens, which is why it is set from the injection path rather than from the CLI. + pair_label: Mutex>, } impl Stats { @@ -148,7 +150,10 @@ impl Stats { let Ok(mut totals) = self.totals.lock() else { return Totals::default(); }; - totals.solved += observations.len() as u64; + totals.solved += observations + .iter() + .filter(|o| o.solved) + .count() as u64; totals.won += observations .iter() .filter(|o| o.won) @@ -158,6 +163,23 @@ impl Stats { *totals } + /// Records the mirrored pair's symbol label, so the report can name it. + pub(crate) fn set_pair_label(&self, label: &str) { + if let Ok(mut pair_label) = self.pair_label.lock() { + if pair_label.is_none() { + *pair_label = Some(label.to_string()); + } + } + } + + /// The mirrored pair's symbol label, once an injection has resolved it. + pub(crate) fn pair_label(&self) -> Option { + self.pair_label + .lock() + .ok() + .and_then(|label| label.clone()) + } + /// The run totals so far. pub(crate) fn totals(&self) -> Totals { self.totals @@ -176,15 +198,63 @@ pub(crate) fn biguint_to_f64(value: &BigUint) -> f64 { .unwrap_or(f64::INFINITY) } +/// The mock-`PropAMM` fields written into a comparison record, so the offline `report` subcommand +/// reads them alongside everything else it already knows about the trade (venue, solver, pair). +/// +/// Amounts are decimal strings, like every other amount in the record: they exceed `f64`'s exact +/// integer range and JSON has no integer type wide enough. +#[derive(Debug, Clone, Serialize)] +pub(crate) struct Record { + /// The mirrored pair as token symbols, e.g. `WETH/USDC` — which pool the mock stood in for. + pub pair: Option, + /// Whether the winning route ran through the mock `PropAMM` pool. + pub won: bool, + /// The public-market output the user is committed to. + pub committed_amount_out: Option, + /// Output the mock produced above the commitment — the fee it could have charged. + pub fee_headroom: Option, + /// That headroom as a fraction of the commitment, in basis points. + pub fee_headroom_bps: Option, + /// The committed output valued in USD — the flow the pool captured. + pub committed_usd: Option, + /// The headroom valued in USD. + pub fee_headroom_usd: Option, +} + +impl Record { + /// Projects an observation into its record, given the USD valuations the caller computed. + pub(crate) fn new( + observed: &Observation, + pair: Option, + committed_usd: Option, + fee_headroom_usd: Option, + ) -> Self { + Self { + pair, + won: observed.won, + committed_amount_out: observed + .committed_amount_out + .as_ref() + .map(std::string::ToString::to_string), + fee_headroom: observed + .fee_headroom + .as_ref() + .map(std::string::ToString::to_string), + fee_headroom_bps: observed.fee_headroom_bps(), + committed_usd, + fee_headroom_usd, + } + } +} + #[cfg(test)] mod tests { use super::*; fn observation(won: bool, committed: u64, headroom: u64) -> Observation { Observation { - token_in: Address::from([0x11; 20]), token_out: Address::from([0x22; 20]), - amount_in: BigUint::from(1_000u64), + solved: true, won, committed_amount_out: won.then(|| BigUint::from(committed)), fee_headroom: won.then(|| BigUint::from(headroom)), diff --git a/tools/hindsight/src/report/aggregate.rs b/tools/hindsight/src/report/aggregate.rs index 793c0ba1..f801b856 100644 --- a/tools/hindsight/src/report/aggregate.rs +++ b/tools/hindsight/src/report/aggregate.rs @@ -7,6 +7,9 @@ use std::collections::HashMap; use crate::report::record::Comparison; +/// Number of order pairs listed in the mock-`PropAMM` breakdown table. +const TOP_PROPAMM_PAIRS: usize = 10; + /// Number of trades listed in the biggest-wins and biggest-losses tables. const TOP_TRADES: usize = 10; /// Number of tokens listed in the unsolvable-tail table. @@ -28,6 +31,59 @@ pub(crate) struct Report { pub top_wins: Vec, pub top_losses: Vec, pub unsolvable_tokens: Vec, + /// Present only for a run the monitor drove with `--propamm-pair`. + pub propamm: Option, +} + +/// The mock-`PropAMM` view: how often the exclusive route beat the public one, and what fee it +/// could have charged on the flow it took. +/// +/// The run-wide winrate is over every solved order, most of which never touch the mirrored pair — +/// so `by_pair` is where the answer actually lives. Each row is one order direction, and the mock +/// can serve an order whose own pair differs from the mirrored one (a `DAI→USDC` order routed +/// `DAI→WETH→USDC` uses a `WETH/USDC` leg), which is exactly what the breakdown shows. +pub(crate) struct PropAmm { + /// The mirrored pair as token symbols, e.g. `WETH/USDC`. + pub pair: Option, + /// Orders with a scored quote — the winrate's denominator. + pub solved: usize, + /// Of those, how many routed through the mock pool. + pub won: usize, + /// Committed output on wins, valued in USD — the flow the pool captured. + pub captured_flow_usd: f64, + /// Fee headroom on wins, valued in USD. + pub fee_headroom_usd: f64, + /// Median fee headroom in bps over wins — the typical fee the pool could charge. + pub median_headroom_bps: Option, + /// Per-order-pair breakdown, most wins first. + pub by_pair: Vec, +} + +impl PropAmm { + /// Fee headroom as a fraction of captured flow, in basis points. `None` when it captured no + /// flow, where the ratio is undefined rather than zero. + pub(crate) fn avg_headroom_bps(&self) -> Option { + (self.captured_flow_usd > 0.0) + .then(|| self.fee_headroom_usd / self.captured_flow_usd * 10_000.0) + } +} + +/// One order direction in the mock-`PropAMM` breakdown. +pub(crate) struct PropAmmPair { + /// `token_in` address. + pub token_in: String, + /// `token_out` address. + pub token_out: String, + /// Solved orders in this direction. + pub solved: usize, + /// Of those, how many the mock pool won. + pub won: usize, + /// Committed output on those wins, valued in USD. + pub captured_flow_usd: f64, + /// Fee headroom on those wins, valued in USD. + pub fee_headroom_usd: f64, + /// Median fee headroom in bps over those wins. + pub median_headroom_bps: Option, } pub(crate) struct Summary { @@ -93,9 +149,119 @@ pub(crate) fn build(records: &[Comparison]) -> Report { top_wins: top_wins(records), top_losses: top_losses(records), unsolvable_tokens: unsolvable_tokens(records), + propamm: propamm(records), } } +/// The mock-`PropAMM` view, or `None` when no record carries one — i.e. the monitor ran without +/// `--propamm-pair`, so the section is omitted rather than rendered empty. +/// +/// Only records with a `propamm` field count toward `solved`: a run that enabled the harness +/// mid-way would otherwise dilute the winrate with trades the mock never saw. +fn propamm(records: &[Comparison]) -> Option { + let scoped: Vec<&Comparison> = records + .iter() + .filter(|r| r.propamm.is_some()) + .collect(); + if scoped.is_empty() { + return None; + } + let wins: Vec<&Comparison> = scoped + .iter() + .copied() + .filter(|r| { + r.propamm + .as_ref() + .is_some_and(|p| p.won) + }) + .collect(); + let mut headroom_bps: Vec = wins + .iter() + .filter_map(|r| { + r.propamm + .as_ref() + .and_then(|p| p.fee_headroom_bps) + }) + .collect(); + Some(PropAmm { + pair: scoped.iter().find_map(|r| { + r.propamm + .as_ref() + .and_then(|p| p.pair.clone()) + }), + solved: scoped.len(), + won: wins.len(), + captured_flow_usd: sum_propamm(&wins, |p| p.committed_usd), + fee_headroom_usd: sum_propamm(&wins, |p| p.fee_headroom_usd), + median_headroom_bps: median(&mut headroom_bps), + by_pair: propamm_by_pair(&scoped), + }) +} + +/// Sums an optional USD field over records, skipping the ones where the token was not priced. +fn sum_propamm( + records: &[&Comparison], + field: impl Fn(&crate::report::record::PropAmm) -> Option, +) -> f64 { + records + .iter() + .filter_map(|r| r.propamm.as_ref().and_then(&field)) + .filter(|value| value.is_finite()) + .sum() +} + +/// Groups the mock-`PropAMM` outcomes by order direction, ranked by wins then by solved count, so +/// the pairs the exclusive route actually served come first. +fn propamm_by_pair(scoped: &[&Comparison]) -> Vec { + let mut grouped: HashMap<(&str, &str), Vec<&Comparison>> = HashMap::new(); + for record in scoped { + grouped + .entry((record.token_in.as_str(), record.token_out.as_str())) + .or_default() + .push(record); + } + + let mut rows: Vec = grouped + .into_iter() + .map(|((token_in, token_out), records)| { + let wins: Vec<&Comparison> = records + .iter() + .copied() + .filter(|r| { + r.propamm + .as_ref() + .is_some_and(|p| p.won) + }) + .collect(); + let mut headroom_bps: Vec = wins + .iter() + .filter_map(|r| { + r.propamm + .as_ref() + .and_then(|p| p.fee_headroom_bps) + }) + .collect(); + PropAmmPair { + token_in: token_in.to_string(), + token_out: token_out.to_string(), + solved: records.len(), + won: wins.len(), + captured_flow_usd: sum_propamm(&wins, |p| p.committed_usd), + fee_headroom_usd: sum_propamm(&wins, |p| p.fee_headroom_usd), + median_headroom_bps: median(&mut headroom_bps), + } + }) + .collect(); + rows.sort_by(|a, b| { + b.won + .cmp(&a.won) + .then_with(|| b.solved.cmp(&a.solved)) + .then_with(|| a.token_in.cmp(&b.token_in)) + }); + rows.truncate(TOP_PROPAMM_PAIRS); + rows +} + fn summary(records: &[Comparison]) -> Summary { let mut blocks: Vec = records .iter() @@ -387,4 +553,101 @@ mod tests { assert_eq!(median(&mut [4.0, 1.0, 3.0, 2.0]), Some(2.5)); assert_eq!(median(&mut []), None); } + + /// A record carrying a mock-`PropAMM` outcome. + fn propamm_record( + block: u64, + token_in: &str, + token_out: &str, + won: bool, + headroom_bps: f64, + committed_usd: f64, + ) -> Comparison { + serde_json::from_value(serde_json::json!({ + "block": block, + "settled_tx": format!("0x{block:064x}"), + "venue": "relay", + "solver": "1inch", + "token_in": token_in, + "token_out": token_out, + "top": { "verdict": "win", "net_bps": 5.0, "settled_value_usd": 1000.0 }, + "propamm": { + "pair": "WETH/USDC", + "won": won, + "fee_headroom_bps": won.then_some(headroom_bps), + "committed_usd": won.then_some(committed_usd), + "fee_headroom_usd": won.then(|| committed_usd * headroom_bps / 10_000.0), + }, + })) + .unwrap() + } + + #[test] + fn test_propamm_absent_without_the_harness() { + // An ordinary monitor run writes no `propamm` field, so the section must be omitted rather + // than rendered as a run where the pool won nothing. + let records = vec![record(1, "relay", "1inch", "win", Some(5.0))]; + assert!(propamm(&records).is_none()); + } + + #[test] + fn test_propamm_counts_only_records_the_harness_saw() { + // Enabling the harness mid-run must not dilute the winrate with trades the mock never saw. + let records = vec![ + record(1, "relay", "1inch", "win", Some(5.0)), + propamm_record(2, "0xweth", "0xusdc", true, 4.0, 1_000.0), + propamm_record(3, "0xweth", "0xusdc", false, 0.0, 0.0), + ]; + let propamm = propamm(&records).expect("some records carry an outcome"); + assert_eq!(propamm.solved, 2); + assert_eq!(propamm.won, 1); + assert_eq!(propamm.pair.as_deref(), Some("WETH/USDC")); + } + + #[test] + fn test_propamm_totals_and_flow_weighted_headroom() { + let records = vec![ + propamm_record(1, "0xweth", "0xusdc", true, 4.0, 1_000.0), + propamm_record(2, "0xweth", "0xusdc", true, 8.0, 3_000.0), + propamm_record(3, "0xweth", "0xusdc", false, 0.0, 0.0), + ]; + let propamm = propamm(&records).expect("outcomes present"); + + assert!((propamm.captured_flow_usd - 4_000.0).abs() < 1e-6); + // 1000 @ 4 bps = 0.40, 3000 @ 8 bps = 2.40. + assert!((propamm.fee_headroom_usd - 2.8).abs() < 1e-6); + assert!((propamm.median_headroom_bps.unwrap() - 6.0).abs() < 1e-6); + // Flow-weighted, not the mean of the two bps values: 2.8 / 4000 = 7 bps. + assert!((propamm.avg_headroom_bps().unwrap() - 7.0).abs() < 1e-6); + } + + #[test] + fn test_propamm_avg_headroom_undefined_without_captured_flow() { + // No flow means the ratio has no denominator; reporting 0 would read as "no headroom". + let records = vec![propamm_record(1, "0xweth", "0xusdc", false, 0.0, 0.0)]; + let propamm = propamm(&records).expect("outcomes present"); + assert!(propamm.avg_headroom_bps().is_none()); + } + + #[test] + fn test_propamm_by_pair_ranks_the_served_pairs_first() { + // The mirrored pair is WETH/USDC, but the mock also serves a DAI→USDC order routed through + // WETH. The pair with wins must outrank an equally-sized pair without any. + let records = vec![ + propamm_record(1, "0xdai", "0xusdc", false, 0.0, 0.0), + propamm_record(2, "0xdai", "0xusdc", false, 0.0, 0.0), + propamm_record(3, "0xweth", "0xusdc", true, 4.0, 1_000.0), + propamm_record(4, "0xweth", "0xusdc", false, 0.0, 0.0), + ]; + let propamm = propamm(&records).expect("outcomes present"); + let rows = &propamm.by_pair; + + assert_eq!(rows.len(), 2); + assert_eq!(rows[0].token_in, "0xweth"); + assert_eq!(rows[0].solved, 2); + assert_eq!(rows[0].won, 1); + assert_eq!(rows[1].token_in, "0xdai"); + assert_eq!(rows[1].won, 0); + assert!(rows[1].median_headroom_bps.is_none()); + } } diff --git a/tools/hindsight/src/report/html.rs b/tools/hindsight/src/report/html.rs index ec69abc8..8842cf6a 100644 --- a/tools/hindsight/src/report/html.rs +++ b/tools/hindsight/src/report/html.rs @@ -9,7 +9,7 @@ use std::fmt::Write as _; use crate::report::aggregate::{ - Count, GroupStats, Report, Savings, Summary, TradeRow, VerdictStat, + Count, GroupStats, PropAmm, PropAmmPair, Report, Savings, Summary, TradeRow, VerdictStat, }; /// Shortest share of a stacked column that gets an inline `12.3%` label. Below it the segment is @@ -53,6 +53,9 @@ pub(crate) fn render(report: &Report, filter: Option<&str>) -> String { let mut html = String::from(HEAD); html.push_str(&hero_section(&report.savings, &report.summary, filter)); html.push_str(&verdict_section(&report.verdicts)); + if let Some(propamm) = report.propamm.as_ref() { + html.push_str(&propamm_section(propamm)); + } html.push_str(&trades_section("Top savings", &report.top_wins)); html.push_str(&trades_section("Biggest losses", &report.top_losses)); html.push_str(&group_section("By solver", &report.by_solver)); @@ -268,6 +271,77 @@ fn token_section(tokens: &[Count]) -> String { section("Unsolved token tail", &table) } +/// The mock-`PropAMM` section: what the exclusive route won, and what fee it could have charged on +/// top of the flow it took. +/// +/// Rendered only for a run driven with `monitor --propamm-pair`. The headline winrate is over every +/// solved order in the run, most of which never touch the mirrored pair — so it reads low by +/// construction, and the per-pair table below is where the real answer is. Both are shown rather +/// than only the flattering one. +fn propamm_section(propamm: &PropAmm) -> String { + let pair = propamm + .pair + .as_deref() + .unwrap_or("unknown pair"); + let stat = |value: &str, label: &str| { + format!( + "
{value}
\ +
{label}
" + ) + }; + let headline = format!( + "
{}{}{}{}{}
", + stat(&pct(propamm.won, propamm.solved), "win rate (all solved orders)"), + stat(&fmt_count(propamm.won), "wins"), + stat(&fmt_usd(propamm.captured_flow_usd), "flow captured"), + stat(&fmt_bps(propamm.median_headroom_bps), "median fee headroom bps"), + stat(&fmt_bps(propamm.avg_headroom_bps()), "fee headroom bps (flow-weighted)"), + ); + let body = format!( + "

Mirrored pool: {}, priced fee-free. \ + Fee headroom is what the signed extension could have charged and still beaten the \ + public market — measured, not assumed. Quotes from this pool are not executable.

\ + {headline}{}", + escape(pair), + propamm_pair_table(&propamm.by_pair), + ); + section("Mock PropAMM (exclusive route)", &body) +} + +/// Per-order-direction breakdown. A row's pair can differ from the mirrored pair: the mock serves +/// any route with a leg it can price, so a `DAI→USDC` order routed through `WETH` shows up here +/// too. +fn propamm_pair_table(pairs: &[PropAmmPair]) -> String { + if pairs.is_empty() { + return "

no orders

".to_string(); + } + let mut table = String::from( + "\ + \ + ", + ); + for pair in pairs { + let _ = write!( + table, + "\ + \ + ", + escape(&pair.token_in), + escape(&pair.token_out), + escape(&short_hash(&pair.token_in)), + escape(&short_hash(&pair.token_out)), + pair.solved, + pair.won, + pct(pair.won, pair.solved), + fmt_usd(pair.captured_flow_usd), + fmt_usd(pair.fee_headroom_usd), + fmt_bps(pair.median_headroom_bps), + ); + } + table.push_str("
order pairsolvedwinswin%flow capturedfee headroommedian headroom bps
{} → {}{}{}{}{}{}{}
"); + table +} + fn section(title: &str, body: &str) -> String { format!("

{}

{body}
", escape(title)) } @@ -400,6 +474,8 @@ section { background: #211a30; border: 1px solid #362b4a; border-radius: 8px; .collab { color: #9a8bbf; font-size: .75rem; text-transform: uppercase; letter-spacing: .04em; text-align: center; } .nodata { color: #9a8bbf; margin: 0; } +/* A section's framing sentence: what the numbers below mean, before they are read. */ +.note { color: #b9adcf; margin: 0 0 .5rem; max-width: 68ch; line-height: 1.55; } table { border-collapse: collapse; width: 100%; } th, td { text-align: left; padding: .4rem .6rem; border-bottom: 1px solid #362b4a; } th { color: #9a8bbf; font-weight: 600; font-size: .8rem; text-transform: uppercase; } @@ -553,4 +629,52 @@ mod tests { fn test_escape_replaces_markup() { assert_eq!(escape("&\"x\""), "<a>&"x""); } + + /// A report from records carrying mock-`PropAMM` outcomes. + fn propamm_report(won: bool) -> Report { + let records: Vec = vec![serde_json::json!({ + "block": 1, + "settled_tx": "0xabc0000000000000000000000000000000000000000000000000000000000001", + "venue": "relay", "solver": "1inch", + "token_in": "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2", + "token_out": "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", + "top": {"verdict": "win", "net_bps": 20.0, "improvement_usd": 12.0, "settled_value_usd": 1000.0}, + "propamm": { + "pair": "WETH/USDC", "won": won, + "fee_headroom_bps": won.then_some(4.0), + "committed_usd": won.then_some(1_000.0), + "fee_headroom_usd": won.then_some(0.4), + }, + })] + .into_iter() + .map(|v| serde_json::from_value(v).unwrap()) + .collect(); + build(&records) + } + + #[test] + fn test_propamm_section_omitted_without_the_harness() { + // An ordinary run's report must look exactly as it did before the harness existed. + assert!(!render(&sample_report(), None).contains("Mock PropAMM")); + } + + #[test] + fn test_propamm_section_names_the_mirrored_pair_and_its_numbers() { + let html = render(&propamm_report(true), None); + assert!(html.contains("Mock PropAMM (exclusive route)")); + assert!(html.contains("WETH/USDC"), "the section must name the pool it stood in for"); + assert!(html.contains("median fee headroom bps")); + // The order pair is rendered short, with the full addresses in the hover title. + assert!(html.contains("0xc02aaa39…756cc2")); + assert!(html.contains("0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2")); + } + + #[test] + fn test_propamm_section_renders_a_run_the_pool_never_won() { + // A zero-win run is a result, not an error: the section still renders, with an em dash + // where the median headroom is undefined. + let html = render(&propamm_report(false), None); + assert!(html.contains("Mock PropAMM (exclusive route)")); + assert!(html.contains("0.0%"), "a zero winrate must be stated, not omitted"); + } } diff --git a/tools/hindsight/src/report/record.rs b/tools/hindsight/src/report/record.rs index 93104803..fe2225e5 100644 --- a/tools/hindsight/src/report/record.rs +++ b/tools/hindsight/src/report/record.rs @@ -19,6 +19,32 @@ pub(crate) struct Comparison { pub token_out: String, /// Optimistic state (N-1); the report's headline, matching the monitor's headline verdict. pub top: State, + /// The mock-`PropAMM` outcome, present only for runs the monitor drove with `--propamm-pair`. + #[serde(default)] + pub propamm: Option, +} + +/// One trade's mock-`PropAMM` outcome, as written by `monitor --propamm-pair`. +/// +/// The mock pool quotes at a configured fee-free price and charges nothing, so `fee_headroom` is +/// the fee the signed extension could have charged on this trade and still beaten the public +/// market. +#[derive(Debug, Clone, Deserialize)] +pub(crate) struct PropAmm { + /// The mirrored pair as token symbols, e.g. `WETH/USDC`. + #[serde(default)] + pub pair: Option, + /// Whether the winning route ran through the mock pool. + pub won: bool, + /// That headroom as a fraction of the committed output, in basis points. + #[serde(default)] + pub fee_headroom_bps: Option, + /// The committed output valued in USD — the flow the pool captured. + #[serde(default)] + pub committed_usd: Option, + /// The headroom valued in USD. + #[serde(default)] + pub fee_headroom_usd: Option, } /// Fynd's result at one block state. @@ -106,7 +132,7 @@ mod tests { let range = build_range(&trade, &prices, top, Outcome::Unsolvable("x".into())); let mut buf = Vec::new(); - write_comparisons(&mut buf, std::slice::from_ref(&range), &prices, &prices); + write_comparisons(&mut buf, std::slice::from_ref(&range), &prices, &prices, &[]); let line = String::from_utf8(buf).unwrap(); let record: Comparison = serde_json::from_str(line.trim()).unwrap(); @@ -119,4 +145,61 @@ mod tests { assert!((record.top.improvement_usd.unwrap() - 10.0).abs() < 1e-3); assert_eq!(record.token_out, format!("{usdc:#x}")); } + + /// The mock-`PropAMM` fields the monitor writes parse back too — the same writer/reader drift + /// guard, for the fields the `PropAMM` section keys off. + #[test] + fn test_parses_the_propamm_fields_from_the_monitor_writer() { + use crate::propamm::report::{Observation, Record}; + + let usdc: Address = "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48" + .parse() + .unwrap(); + let observed = Observation { + token_out: usdc, + solved: true, + won: true, + committed_amount_out: Some(1_000_000_000u64.into()), + fee_headroom: Some(400_000u64.into()), + }; + let record = + Record::new(&observed, Some("WETH/USDC".to_string()), Some(1_000.0), Some(0.4)); + + let line = serde_json::to_string(&serde_json::json!({ + "block": 1, + "settled_tx": "0xabc", + "venue": "relay", + "solver": "1inch", + "token_in": "0xaaa", + "token_out": "0xbbb", + "top": { "verdict": "win" }, + "propamm": record, + })) + .unwrap(); + + let parsed: Comparison = serde_json::from_str(&line).unwrap(); + let propamm = parsed + .propamm + .expect("the propamm field round-trips"); + assert_eq!(propamm.pair.as_deref(), Some("WETH/USDC")); + assert!(propamm.won); + // 400_000 / 1_000_000_000 = 4 bps. + assert!((propamm.fee_headroom_bps.unwrap() - 4.0).abs() < 1e-9); + assert!((propamm.committed_usd.unwrap() - 1_000.0).abs() < 1e-9); + assert!((propamm.fee_headroom_usd.unwrap() - 0.4).abs() < 1e-9); + } + + /// An ordinary monitor run writes no `propamm` field, and the reader must treat that as absent + /// rather than failing to parse the whole record. + #[test] + fn test_propamm_is_absent_when_the_harness_is_off() { + let line = serde_json::json!({ + "block": 1, "settled_tx": "0xabc", "venue": "relay", "solver": "1inch", + "token_in": "0xaaa", "token_out": "0xbbb", "top": { "verdict": "win" }, + "propamm": serde_json::Value::Null, + }) + .to_string(); + let parsed: Comparison = serde_json::from_str(&line).unwrap(); + assert!(parsed.propamm.is_none()); + } } diff --git a/tools/hindsight/src/resolve/jsonl.rs b/tools/hindsight/src/resolve/jsonl.rs index f5d40c7a..af0d46be 100644 --- a/tools/hindsight/src/resolve/jsonl.rs +++ b/tools/hindsight/src/resolve/jsonl.rs @@ -118,14 +118,21 @@ fn date_from_unix(secs: u64) -> String { /// filter to wins for the improvement view or to unsolvables for the coverage worklist (where Fynd /// needs to improve). Losses keep their route (what path Fynd took and lost on); unsolvables keep /// the reason. +/// `propamm` carries one optional mock-`PropAMM` outcome per range, in the same order; a shorter +/// slice simply leaves the trailing records without a `propamm` field. pub(crate) fn write_comparisons( writer: &mut W, ranges: &[RangeComparison], prices_top: &Prices, prices_back: &Prices, + propamm: &[Option], ) { - for range in ranges { - let Ok(line) = serde_json::to_string(&comparison_record(range, prices_top, prices_back)) + for (index, range) in ranges.iter().enumerate() { + let propamm = propamm + .get(index) + .and_then(Option::as_ref); + let Ok(line) = + serde_json::to_string(&comparison_record(range, prices_top, prices_back, propamm)) else { continue; }; @@ -146,6 +153,7 @@ fn comparison_record( range: &RangeComparison, prices_top: &Prices, prices_back: &Prices, + propamm: Option<&crate::propamm::report::Record>, ) -> serde_json::Value { serde_json::json!({ "block": range.block_number, @@ -167,6 +175,9 @@ fn comparison_record( "sandwich": range.sandwich, "top": state_record(&range.top, range, prices_top), "back": state_record(&range.back, range, prices_back), + // Absent unless the run had `--propamm-pair` set, so an ordinary run's records are + // byte-identical to before. + "propamm": propamm, }) } @@ -338,7 +349,7 @@ mod tests { Outcome::Unsolvable("x".into()), Outcome::Unsolvable("x".into()), ); - let rec = comparison_record(&range, &empty_prices(), &empty_prices()); + let rec = comparison_record(&range, &empty_prices(), &empty_prices(), None); assert_eq!(rec.pointer("/tx_index").unwrap(), 3); assert_eq!( rec.pointer("/quoted_amount_out") @@ -428,7 +439,7 @@ mod tests { }); let range = build_range(&trade, &prices, top, back); - let rec = comparison_record(&range, &prices, &prices); + let rec = comparison_record(&range, &prices, &prices, None); let top_usd = rec .pointer("/top/improvement_usd") .unwrap() @@ -494,7 +505,7 @@ mod tests { Outcome::Unsolvable("missing token in Tycho".into()), Outcome::Unsolvable("missing token in Tycho".into()), ); - let rec = comparison_record(&range, &empty_prices(), &empty_prices()); + let rec = comparison_record(&range, &empty_prices(), &empty_prices(), None); assert_eq!(rec.pointer("/top/verdict").unwrap(), "unsolvable"); assert_eq!( rec.pointer("/top/unsolvable_reason") @@ -545,7 +556,7 @@ mod tests { }) }; let range = build_range(&trade, &empty_prices(), solved(1_100), solved(1_050)); - let rec = comparison_record(&range, &empty_prices(), &empty_prices()); + let rec = comparison_record(&range, &empty_prices(), &empty_prices(), None); assert_eq!(rec.pointer("/tx_index").unwrap(), 42); assert_eq!(rec.pointer("/top/verdict").unwrap(), "sandwiched"); diff --git a/tools/hindsight/src/resolve/monitor.rs b/tools/hindsight/src/resolve/monitor.rs index 08c12fd4..2705fd34 100644 --- a/tools/hindsight/src/resolve/monitor.rs +++ b/tools/hindsight/src/resolve/monitor.rs @@ -33,7 +33,7 @@ use tycho_simulation::tycho_common::models::{Address as CoreAddress, Chain}; use crate::{ decoder::{DecodedTrade, Decoder, Registry}, propamm, provider_from, - resolve::{resolve_block_range, Outcome, SolvedAmount, SteppingSolver}, + resolve::{resolve_block_range, Outcome, RangeComparison, SolvedAmount, SteppingSolver}, telemetry, usd::Prices, }; @@ -140,11 +140,6 @@ pub(crate) struct MonitorArgs { /// actually prices those trades best #[arg(long, env = "PROPAMM_PROBE_UNITS", default_value_t = 1.0)] pub propamm_probe_units: f64, - - /// Append one JSON line per re-solved order that the mock `PropAMM` won, with the committed - /// output, the fee headroom in bps, and both valued in USD - #[arg(long, env = "PROPAMM_JSONL")] - pub propamm_jsonl: Option, } /// The mock-`PropAMM` scaffold attached to one monitor run. @@ -154,7 +149,6 @@ pub(crate) struct MonitorArgs { struct PropAmmHarness { injector: tokio::sync::Mutex, stats: Arc, - jsonl: Option, } /// Drives the in-process solver, stepping the chain one block per `SteppingSolver::advance`. @@ -242,14 +236,19 @@ impl StepAdapter<'_> { .inject(self.solver, block) .await { - Ok(Some(injected)) => debug!( - block, - source = injected.source_component, - source_price = injected.source_price, - mock_price = injected.mock_price, - derived_data_ready = injected.derived_data_ready, - "mirrored the mock PropAMM pool" - ), + Ok(Some(injected)) => { + harness + .stats + .set_pair_label(&injected.pair_label); + debug!( + block, + source = injected.source_component, + source_price = injected.source_price, + mock_price = injected.mock_price, + derived_data_ready = injected.derived_data_ready, + "mirrored the mock PropAMM pool" + ); + } Ok(None) => warn!( block, "no source pool for the mirrored pair carries state yet; the mock PropAMM is \ @@ -273,7 +272,7 @@ impl SteppingSolver for StepAdapter<'_> { let order = Order::new( CoreAddress::from(token_in.into_array()), CoreAddress::from(token_out.into_array()), - amount.clone(), + amount, OrderSide::Sell, CoreAddress::from([0x11u8; 20]), ); @@ -290,18 +289,12 @@ impl SteppingSolver for StepAdapter<'_> { return Outcome::Unsolvable("solver returned no order quote".to_string()); }; if let Some(harness) = self.propamm { - // Only successful quotes count, so the winrate's denominator is orders the - // PropAMM actually had a chance at. - if order_quote.status() == QuoteStatus::Success { - harness - .stats - .record(propamm::report::Observation::from_quote( - order_quote, - token_in, - token_out, - amount, - )); - } + // Every solve is recorded, successful or not, so the sink stays index-aligned + // with the block's trades; `Observation::solved` is what the winrate's + // denominator counts. + harness + .stats + .record(propamm::report::Observation::from_quote(order_quote, token_out)); } order_quote_to_outcome(order_quote) } @@ -602,7 +595,6 @@ pub(crate) async fn run(cfg: MonitorArgs) -> anyhow::Result<()> { .map(|config| PropAmmHarness { injector: tokio::sync::Mutex::new(propamm::Injector::new(&solver, config)), stats: Arc::clone(&propamm_stats), - jsonl: cfg.propamm_jsonl.clone(), }); let adapter = StepAdapter { solver: &solver, @@ -705,22 +697,34 @@ fn log_propamm_summary(config: &propamm::MirrorConfig, totals: &propamm::report: } /// Values one block's mock-`PropAMM` observations in USD, folds them into the run totals, logs the -/// running picture, and appends the wins to JSONL. +/// running picture, and returns one record per trade for the comparisons JSONL. /// /// Amounts are valued at top-of-block prices, matching the headline improvement metric. Both halves /// are valued: the committed output answers "how much flow the pool captured", the headroom answers /// "how much fee it could have charged on that flow", and their ratio is that fee in bps. +/// +/// `resolve_block_range` solves every trade at top-of-block in order, advances, then solves every +/// trade again at back-of-block in the same order — so the sink holds `2 * trade_count` +/// observations and the first half are the top-of-block ones the report keys off. A different count +/// means that pairing no longer holds, so the records are dropped rather than misattributed; the +/// totals are unaffected either way. fn report_propamm_block( harness: &PropAmmHarness, block: u64, prices: &Prices, observations: &[propamm::report::Observation], -) { + trade_count: usize, +) -> Vec> { let mut headroom_usd = 0.0; let mut captured_flow_usd = 0.0; - let mut lines = Vec::new(); + let mut records: Vec> = Vec::with_capacity(observations.len()); + let pair_label = harness.stats.pair_label(); - for observed in observations.iter().filter(|o| o.won) { + for observed in observations { + if !observed.won { + records.push(None); + continue; + } let value_usd = |amount: Option<&BigUint>| { amount .and_then(biguint_to_u256_opt) @@ -730,26 +734,12 @@ fn report_propamm_block( let committed = value_usd(observed.committed_amount_out.as_ref()); headroom_usd += headroom.unwrap_or(0.0); captured_flow_usd += committed.unwrap_or(0.0); - - if harness.jsonl.is_some() { - lines.push(serde_json::json!({ - "block": block, - "token_in": observed.token_in.to_string(), - "token_out": observed.token_out.to_string(), - "amount_in": observed.amount_in.to_string(), - "committed_amount_out": observed - .committed_amount_out - .as_ref() - .map(std::string::ToString::to_string), - "fee_headroom": observed - .fee_headroom - .as_ref() - .map(std::string::ToString::to_string), - "fee_headroom_bps": observed.fee_headroom_bps(), - "committed_usd": committed, - "fee_headroom_usd": headroom, - })); - } + records.push(Some(propamm::report::Record::new( + observed, + pair_label.clone(), + committed, + headroom, + ))); } let totals = harness @@ -761,7 +751,7 @@ fn report_propamm_block( .count(); info!( block, - block_orders = observations.len(), + block_solves = observations.len(), block_wins, block_fee_headroom_usd = format!("{headroom_usd:.2}"), run_winrate_pct = format!("{:.1}", totals.winrate_pct()), @@ -770,28 +760,20 @@ fn report_propamm_block( "mock PropAMM block result" ); - if let Some(path) = harness.jsonl.as_ref() { - if let Err(e) = append_jsonl(path, &lines) { - warn!(path = %path.display(), "failed to append PropAMM JSONL: {e}"); - } - } -} - -/// Appends one JSON object per line, creating the file if absent. -fn append_jsonl(path: &std::path::Path, lines: &[serde_json::Value]) -> std::io::Result<()> { - use std::io::Write as _; - - if lines.is_empty() { - return Ok(()); + if records.len() == trade_count * 2 { + records.truncate(trade_count); + return records; } - let mut file = std::fs::OpenOptions::new() - .create(true) - .append(true) - .open(path)?; - for line in lines { - writeln!(file, "{line}")?; + if !observations.is_empty() { + warn!( + block, + solves = observations.len(), + trades = trade_count, + "PropAMM observations do not pair with the block's trades; omitting them from the \ + comparisons JSONL" + ); } - file.flush() + vec![None; trade_count] } /// Converts a `BigUint` to `U256`, or `None` when it does not fit — the amount is then left @@ -806,6 +788,51 @@ fn biguint_to_u256_opt(value: &BigUint) -> Option { Some(U256::from_be_bytes(buf)) } +/// Record one block's re-solved trades: Prometheus metrics, the mock-`PropAMM` roll-up, and the +/// comparisons JSONL. +/// +/// The `PropAMM` outcomes are drained before the comparisons are written, so each range carries its +/// outcome into the same JSONL record the offline `report` subcommand reads. They are drained +/// unconditionally when the harness is on, so a block whose trades all failed to solve still clears +/// the sink rather than carrying observations into the next block. +#[expect(clippy::too_many_arguments)] +fn emit_block_results( + cfg: &MonitorArgs, + adapter: &StepAdapter<'_>, + decoder: &Decoder

, + comparisons: &mut Option, + block: u64, + ranges: &[RangeComparison], + prices_top: &Prices, + prices_back: &Prices, +) { + for range in ranges { + telemetry::record_range( + range, + &cfg.chain.name, + prices_top, + prices_back, + decoder.registry(), + ); + } + let propamm_records = match adapter.propamm { + Some(harness) => { + let observations = harness.stats.drain(); + report_propamm_block(harness, block, prices_top, &observations, ranges.len()) + } + None => vec![None; ranges.len()], + }; + if let Some(rotating) = comparisons.as_mut() { + super::jsonl::write_comparisons( + rotating.writer(), + ranges, + prices_top, + prices_back, + &propamm_records, + ); + } +} + /// Drive one solver session: step blocks and re-solve each block's settled trades until the run /// completes or the feed dies. async fn run_session( @@ -899,24 +926,16 @@ async fn run_session( "back-of-block state is not the target block; back comparison may be off" ); } - for range in &ranges { - telemetry::record_range( - range, - &cfg.chain.name, - &prices_top, - &prices_back, - decoder.registry(), - ); - } - if let Some(rotating) = comparisons.as_mut() { - super::jsonl::write_comparisons(rotating.writer(), &ranges, &prices_top, &prices_back); - } - // Drained unconditionally when the harness is on, so a block whose trades all failed to - // solve still clears the sink rather than carrying observations into the next block. - if let Some(harness) = adapter.propamm { - let observations = harness.stats.drain(); - report_propamm_block(harness, target, &prices_top, &observations); - } + emit_block_results( + cfg, + adapter, + decoder, + comparisons, + target, + &ranges, + &prices_top, + &prices_back, + ); let elapsed_s = start.elapsed().as_secs_f64(); telemetry::record_block_seconds(elapsed_s); @@ -1067,7 +1086,6 @@ mod tests { propamm_pair: None, propamm_price_pct: 100.05, propamm_probe_units: 1.0, - propamm_jsonl: None, }) .await .expect("monitor should process one block without error"); From fad87bf457ea35d0566ba0935b88b880ac5ce06a Mon Sep 17 00:00:00 2001 From: TAMARA LIPOWSKI Date: Tue, 28 Jul 2026 13:57:08 -0400 Subject: [PATCH 05/12] fix(hindsight): record every solve so PropAMM outcomes stay aligned A solve that never produced a quote was skipped, so the positional join onto the block's comparison records shifted every later observation onto the wrong trade and dropped the block's outcomes entirely. Record on all paths; only successful quotes count toward the winrate. --- tools/hindsight/CLAUDE.md | 10 +++++ tools/hindsight/src/propamm/report.rs | 40 ++++++++++++++++++++ tools/hindsight/src/resolve/monitor.rs | 51 ++++++++++++++++++-------- 3 files changed, 86 insertions(+), 15 deletions(-) diff --git a/tools/hindsight/CLAUDE.md b/tools/hindsight/CLAUDE.md index 0c946eaf..9174260d 100644 --- a/tools/hindsight/CLAUDE.md +++ b/tools/hindsight/CLAUDE.md @@ -135,6 +135,16 @@ the mock never changes hindsight's own win/loss verdicts — it only adds this s Requires `fynd-core`'s `experimental` feature for `Solver::market_event_sender`. Not for production: the mock prices a pool that does not exist on chain, so any calldata it produces is unexecutable. +Two things to get right when running it: + +- **Set `EXCLUSIVE_SWAP_CONTROLLER_KEY`** (any throwaway key — nothing is executed). The encoder + fails fast on an exclusive leg with no signer, which turns every win into a failed quote and + silently reports a zero winrate. +- **Pick a pair that has flow, and check the routes.** The exclusive leg must be terminal in its + path and within the pool's hop budget, so a pair absent from the block's settled trades yields + zero wins for reasons that have nothing to do with the price. Sanity-check with a deliberately + absurd `--propamm-price-pct 105`: the measured `fee_headroom_bps` should come back at ~500. + ### Verdict model Each re-solved trade produces a `top` (optimistic, state N-1) and `back` (pessimistic, state N) diff --git a/tools/hindsight/src/propamm/report.rs b/tools/hindsight/src/propamm/report.rs index 66f5e429..33185b90 100644 --- a/tools/hindsight/src/propamm/report.rs +++ b/tools/hindsight/src/propamm/report.rs @@ -62,6 +62,21 @@ impl Observation { } } + /// An observation for a solve that never produced a quote. + /// + /// Recorded so the sink stays index-aligned with the block's trades — the join back to each + /// comparison record is positional, so a skipped solve would shift every later observation onto + /// the wrong trade. + pub(crate) fn unsolved(token_out: Address) -> Self { + Self { + token_out, + solved: false, + won: false, + committed_amount_out: None, + fee_headroom: None, + } + } + /// The fee the mock could have charged, as a fraction of the committed output, in basis points. /// /// This is the headline per-trade number: "the pool could have taken this much fee and the user @@ -320,4 +335,29 @@ mod tests { assert_eq!(stats.drain().len(), 2); assert!(stats.drain().is_empty(), "a second drain must not repeat observations"); } + + #[test] + fn test_unsolved_observation_counts_toward_neither_total() { + // A solve that never produced a quote is recorded to keep the sink index-aligned, but it is + // not an order the pool had a chance at — so it must not enter the winrate's denominator. + let stats = Stats::default(); + let totals = stats.accumulate( + &[Observation::unsolved(Address::from([0x22; 20])), observation(true, 100, 1)], + 0.5, + 100.0, + ); + assert_eq!(totals.solved, 1); + assert_eq!(totals.won, 1); + } + + #[test] + fn test_pair_label_is_set_once_and_read_back() { + let stats = Stats::default(); + assert!(stats.pair_label().is_none()); + stats.set_pair_label("WETH/USDC"); + // The mirrored pool can change block to block; the pair it prices cannot, so the first + // label sticks rather than churning. + stats.set_pair_label("DAI/USDC"); + assert_eq!(stats.pair_label().as_deref(), Some("WETH/USDC")); + } } diff --git a/tools/hindsight/src/resolve/monitor.rs b/tools/hindsight/src/resolve/monitor.rs index 2705fd34..92260a59 100644 --- a/tools/hindsight/src/resolve/monitor.rs +++ b/tools/hindsight/src/resolve/monitor.rs @@ -259,11 +259,17 @@ impl StepAdapter<'_> { } } -#[async_trait] -impl SteppingSolver for StepAdapter<'_> { - async fn solve(&self, token_in: Address, token_out: Address, amount_in: U256) -> Outcome { +impl StepAdapter<'_> { + /// Quote one sell order at the solver's current state, returning the outcome and — when a quote + /// came back at all — its mock-`PropAMM` observation. + async fn quote_order( + &self, + token_in: Address, + token_out: Address, + amount_in: U256, + ) -> (Outcome, Option) { let Ok(amount) = amount_in.to_string().parse::() else { - return Outcome::Unsolvable("unparseable amount_in".to_string()); + return (Outcome::Unsolvable("unparseable amount_in".to_string()), None); }; // Placeholder receiver: routing/amounts are receiver-independent; it only fills the encoded // calldata's recipient. Encoding is requested so each quote carries its on-chain @@ -286,20 +292,35 @@ impl SteppingSolver for StepAdapter<'_> { match self.solver.quote(request).await { Ok(quote) => { let Some(order_quote) = quote.orders().first() else { - return Outcome::Unsolvable("solver returned no order quote".to_string()); + return ( + Outcome::Unsolvable("solver returned no order quote".to_string()), + None, + ); }; - if let Some(harness) = self.propamm { - // Every solve is recorded, successful or not, so the sink stays index-aligned - // with the block's trades; `Observation::solved` is what the winrate's - // denominator counts. - harness - .stats - .record(propamm::report::Observation::from_quote(order_quote, token_out)); - } - order_quote_to_outcome(order_quote) + let observed = propamm::report::Observation::from_quote(order_quote, token_out); + (order_quote_to_outcome(order_quote), Some(observed)) } - Err(e) => Outcome::Unsolvable(format!("solve error: {e}")), + Err(e) => (Outcome::Unsolvable(format!("solve error: {e}")), None), + } + } +} + +#[async_trait] +impl SteppingSolver for StepAdapter<'_> { + async fn solve(&self, token_in: Address, token_out: Address, amount_in: U256) -> Outcome { + let (outcome, observed) = self + .quote_order(token_in, token_out, amount_in) + .await; + // Recorded on every path, including the ones that never produced a quote: the sink is + // joined back to the block's trades by position, so a skipped solve would shift + // every later observation onto the wrong trade. `Observation::solved` is what the + // winrate counts. + if let Some(harness) = self.propamm { + harness.stats.record( + observed.unwrap_or_else(|| propamm::report::Observation::unsolved(token_out)), + ); } + outcome } async fn advance(&self) -> anyhow::Result<()> { From 4427c146eb662f9c99edf8c32aff4eb143bfdb0a Mon Sep 17 00:00:00 2001 From: TAMARA LIPOWSKI Date: Tue, 28 Jul 2026 14:22:56 -0400 Subject: [PATCH 06/12] fix(hindsight): write a PropAMM record for every solved order Only wins carried a record, so the report's denominator was wins-only and every run rendered a 100% win rate. Losses now carry a record too; unsolved solves stay absent, since the pool never had a chance at those. --- tools/hindsight/src/report/aggregate.rs | 3 ++- tools/hindsight/src/resolve/monitor.rs | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/hindsight/src/report/aggregate.rs b/tools/hindsight/src/report/aggregate.rs index f801b856..7c1a5e7b 100644 --- a/tools/hindsight/src/report/aggregate.rs +++ b/tools/hindsight/src/report/aggregate.rs @@ -157,7 +157,8 @@ pub(crate) fn build(records: &[Comparison]) -> Report { /// `--propamm-pair`, so the section is omitted rather than rendered empty. /// /// Only records with a `propamm` field count toward `solved`: a run that enabled the harness -/// mid-way would otherwise dilute the winrate with trades the mock never saw. +/// mid-way would otherwise dilute the winrate with trades the mock never saw. The monitor writes +/// the field for every *solved* order, win or lose, so the field's presence is the denominator. fn propamm(records: &[Comparison]) -> Option { let scoped: Vec<&Comparison> = records .iter() diff --git a/tools/hindsight/src/resolve/monitor.rs b/tools/hindsight/src/resolve/monitor.rs index 92260a59..1eadd538 100644 --- a/tools/hindsight/src/resolve/monitor.rs +++ b/tools/hindsight/src/resolve/monitor.rs @@ -742,7 +742,10 @@ fn report_propamm_block( let pair_label = harness.stats.pair_label(); for observed in observations { - if !observed.won { + // A record is written for every solved order, not only the wins: the report needs the + // losses as the winrate's denominator, and a wins-only file makes every run look + // like 100%. Unsolved solves stay `None` — the pool never had a chance at those. + if !observed.solved { records.push(None); continue; } From 6c7b29f666960e41a9e8a62ba8848fbd1c1df205 Mon Sep 17 00:00:00 2001 From: TAMARA LIPOWSKI Date: Tue, 28 Jul 2026 16:29:35 -0400 Subject: [PATCH 07/12] feat(hindsight): price the mock PropAMM against the public best route Each order on the mirrored pair is now solved twice: once with the mock neutralised, giving the route Fynd would otherwise have quoted, then again with the mock rescaled to land a configured offset off that output. So an offset means what it says, and every calibrated order becomes an assertion rather than a data point. --propamm-offsets-bps replaces --propamm-price-pct. Orders cycle through the ladder, and the report judges each group against the behaviour its price implies: below market must never be selected, at market can only win on gas and must take no fee, above market must not take more fee than the offset. --- tools/hindsight/src/propamm/mirror.rs | 135 +++++++++++ tools/hindsight/src/propamm/mod.rs | 188 +++++++++++++-- tools/hindsight/src/propamm/report.rs | 38 ++- tools/hindsight/src/report/aggregate.rs | 296 ++++++++++++++++++++++++ tools/hindsight/src/report/html.rs | 147 +++++++++++- tools/hindsight/src/report/record.rs | 8 + tools/hindsight/src/resolve/monitor.rs | 164 +++++++++++-- 7 files changed, 937 insertions(+), 39 deletions(-) diff --git a/tools/hindsight/src/propamm/mirror.rs b/tools/hindsight/src/propamm/mirror.rs index 914807c9..34648478 100644 --- a/tools/hindsight/src/propamm/mirror.rs +++ b/tools/hindsight/src/propamm/mirror.rs @@ -51,6 +51,49 @@ impl MirrorPool { Self { inner, price_ppm: price_pct_to_ppm(price_pct) } } + /// Wraps `inner` at an explicit parts-per-million scale, clamped to the same range. + /// + /// Used by the per-order calibration, which solves for the scale that lands the mock's output + /// on a target rather than expressing it as a percentage. + pub(crate) fn from_scale_ppm(inner: Box, scale_ppm: u32) -> Self { + Self { inner, price_ppm: scale_ppm.min(4 * PRICE_SCALE) } + } + + /// The scale that makes this pool's output for `amount_in` land on `target_out`, in parts per + /// million, or `None` when `inner` cannot price the pair. + /// + /// Rounds down, so the calibrated output never overshoots the target — an overshoot at the + /// zero-offset group would manufacture surplus the test is asserting is absent. + pub(crate) fn scale_for_target( + inner: &dyn ProtocolSim, + amount_in: BigUint, + token_in: &Token, + token_out: &Token, + target_out: &BigUint, + ) -> Option { + let unscaled = inner + .get_amount_out(amount_in, token_in, token_out) + .ok()? + .amount; + if unscaled == BigUint::ZERO { + return None; + } + let ppm = target_out * BigUint::from(PRICE_SCALE) / unscaled; + u32::try_from(ppm) + .ok() + .map(|ppm| ppm.min(4 * PRICE_SCALE)) + } + + /// The scale that makes this pool quote essentially nothing, so the router can never select it. + /// + /// One part per million of the mirrored price, not zero: a zero scale makes `spot_price` error, + /// which would keep the pool out of the graph entirely rather than merely out of the winning + /// route — and the point of neutralising it is to read the public best route *with the mock's + /// edge still present*. + pub(crate) const fn neutral_scale_ppm() -> u32 { + 1 + } + /// The mock's fee-free price as a fraction of the mirrored pool's price. pub(crate) fn price_factor(&self) -> f64 { f64::from(self.price_ppm) / f64::from(PRICE_SCALE) @@ -385,4 +428,96 @@ mod tests { .expect("mirror quotes"); assert_eq!(result.gas, BigUint::from(100_000u64)); } + + #[test] + fn test_scale_for_target_lands_the_output_on_the_target() { + // The whole calibration rests on this: solve for the scale, apply it, land on the target. + let source = FlatPool { rate: 1_000 }; + let amount = BigUint::from(1_000u64); + for offset_bps in [-50i64, -5, 0, 5, 50, 500] { + let unscaled = BigUint::from(1_000_000u64); + // Every offset in the table is well above -10_000 bps, so the sum is positive. + let shifted = (10_000 + offset_bps).unsigned_abs(); + let target = &unscaled * BigUint::from(shifted) / BigUint::from(10_000u64); + let scale = MirrorPool::scale_for_target( + &source, + amount.clone(), + &token("A"), + &token("B"), + &target, + ) + .expect("a flat pool can always be scaled"); + + let realized = MirrorPool::from_scale_ppm(Box::new(source.clone()), scale) + .get_amount_out(amount.clone(), &token("A"), &token("B")) + .expect("scaled pool quotes") + .amount; + // Rounding down twice can cost a couple of atomic units on a 1e6 output; the offset it + // represents is far below the 0.5 bps the report tolerates. + let realized = i128::try_from(realized).unwrap(); + let target = i128::try_from(target).unwrap(); + assert!( + (realized - target).abs() <= 2, + "offset {offset_bps}: landed on {realized}, wanted {target}" + ); + } + } + + #[test] + fn test_scale_for_target_rounds_down_so_it_never_overshoots() { + // An overshoot at the at-market group would manufacture surplus the test asserts is absent. + let source = FlatPool { rate: 3 }; + let amount = BigUint::from(7u64); + // Unscaled output is 21; ask for 10, which is not a whole number of parts per million of + // it. + let target = BigUint::from(10u64); + let scale = MirrorPool::scale_for_target( + &source, + amount.clone(), + &token("A"), + &token("B"), + &target, + ) + .unwrap(); + let realized = MirrorPool::from_scale_ppm(Box::new(source), scale) + .get_amount_out(amount, &token("A"), &token("B")) + .unwrap() + .amount; + assert!(realized <= target, "{realized} overshot {target}"); + } + + #[test] + fn test_neutral_scale_quotes_essentially_nothing_but_still_prices() { + // Neutralising must keep the pool in the graph — the point is to read the public best route + // with the mock's edge present — so spot_price has to stay finite. + let neutral = MirrorPool::from_scale_ppm( + Box::new(FlatPool { rate: 1_000 }), + MirrorPool::neutral_scale_ppm(), + ); + let out = neutral + .get_amount_out(BigUint::from(1_000u64), &token("A"), &token("B")) + .expect("still quotes") + .amount; + // One part per million of 1_000_000. + assert_eq!(out, BigUint::from(1u64)); + let price = neutral + .spot_price(&token("A"), &token("B")) + .expect("neutralised pool still has a finite price"); + assert!(price.is_finite() && price > 0.0, "got {price}"); + } + + #[test] + fn test_scale_for_target_declines_a_source_that_cannot_price() { + // A source quoting zero has no scale that reaches a positive target; the caller must leave + // the order uncalibrated rather than divide by zero. + let dead = FlatPool { rate: 0 }; + assert!(MirrorPool::scale_for_target( + &dead, + BigUint::from(1_000u64), + &token("A"), + &token("B"), + &BigUint::from(500u64), + ) + .is_none()); + } } diff --git a/tools/hindsight/src/propamm/mod.rs b/tools/hindsight/src/propamm/mod.rs index 0c7d1844..87b994df 100644 --- a/tools/hindsight/src/propamm/mod.rs +++ b/tools/hindsight/src/propamm/mod.rs @@ -89,16 +89,20 @@ const DERIVED_DATA_WAIT: Duration = Duration::from_secs(30); /// Poll interval while waiting for that recomputation. const DERIVED_DATA_POLL: Duration = Duration::from_millis(25); -/// Which pool to mirror, and at what fee-free price. +/// Which pool to mirror, and how far off the public best route to price it. #[derive(Debug, Clone)] pub(crate) struct MirrorConfig { /// First token of the mirrored pair. pub token_a: Address, /// Second token of the mirrored pair. pub token_b: Address, - /// The mock's fee-free price as a percentage of the best real pool's price for the pair. - /// `100.0` mirrors that pool exactly and must never win — the harness's control case. - pub price_pct: f64, + /// Price offsets, in basis points relative to the public best route's output for the order + /// being solved. Orders on the mirrored pair cycle through these, so one run fills every + /// group. + /// + /// A negative offset prices the mock below the public market and must never be selected; zero + /// matches it exactly; a positive offset is the underbid the pool would offer. + pub offsets_bps: Vec, /// `token_a` amount, in whole units, used to rank candidate source pools each block. Picking /// by realized output at a representative size, rather than by TVL, keeps the mirror on /// the pool that actually prices best for the sizes being re-solved. @@ -120,6 +124,26 @@ impl MirrorConfig { .map_err(|e| anyhow::anyhow!("invalid token address {token_b}: {e}"))?, )) } + + /// Whether an order's tokens are exactly the mirrored pair, in either direction. + /// + /// Only these orders can be calibrated: the mock then serves the whole order in one hop, so its + /// route output is a known function of its scale. A multi-hop route's output is not, which is + /// why off-pair orders are solved normally and carry no offset label. + pub(crate) fn serves(&self, token_in: &Address, token_out: &Address) -> bool { + (*token_in == self.token_a && *token_out == self.token_b) || + (*token_in == self.token_b && *token_out == self.token_a) + } + + /// The offset this order should be priced at, cycling through `offsets_bps` by order index so + /// every group fills at the same rate. `None` when no offsets are configured. + pub(crate) fn offset_for(&self, order_index: u64) -> Option { + if self.offsets_bps.is_empty() { + return None; + } + let index = usize::try_from(order_index % self.offsets_bps.len() as u64).ok()?; + self.offsets_bps.get(index).copied() + } } /// What one injection did, for logging. @@ -127,11 +151,9 @@ impl MirrorConfig { pub(crate) struct Injected { /// Component id of the pool that was mirrored this block. pub source_component: String, - /// The best real pool's price: `token_b` per whole unit of `token_a`, at the probe size. + /// The mirrored pool's price: `token_b` per whole unit of `token_a`, at the probe size. /// Watching this across blocks is how you tell a live mirror from one stuck on a stale state. pub source_price: f64, - /// The mock's fee-free price, i.e. `source_price` scaled by `--propamm-price-pct`. - pub mock_price: f64, /// Whether the mock's spot price was recomputed before the wait expired. pub derived_data_ready: bool, /// The mirrored pair as token symbols, e.g. `WETH/USDC`. @@ -148,6 +170,54 @@ pub(crate) struct Injector { /// Whether the mock has been announced as an added component. Announcing twice would re-add an /// existing graph edge. announced: bool, + /// This block's mirrored pool, kept so per-order calibration can simulate it at the order's + /// own size without re-reading the market. + source: Option, +} + +/// The mirrored pool and its tokens, cached for the current block. +struct CachedSource { + state: Box, + token_a: Token, + token_b: Token, +} + +impl CachedSource { + /// The pair's tokens ordered as `(in, out)` for an order, or `None` if the order is off-pair. + fn order_tokens(&self, token_in: &Address) -> Option<(&Token, &Token)> { + if *token_in == self.token_a.address { + Some((&self.token_a, &self.token_b)) + } else if *token_in == self.token_b.address { + Some((&self.token_b, &self.token_a)) + } else { + None + } + } +} + +/// What one per-order calibration set up. +#[derive(Debug)] +pub(crate) struct Calibration { + /// The public best route's output this offset was measured against. + pub public_best_out: BigUint, + /// The output the mock is expected to produce: `public_best_out` shifted by the offset. + pub target_out: BigUint, + /// The offset applied, in basis points. + pub offset_bps: i32, +} + +impl Calibration { + /// The offset the mock's target actually lands on, in basis points — the calibration's own + /// error check. Integer division and the source's own rounding mean this can differ from + /// `offset_bps` by a fraction of a bps; a large gap means the source could not hold the + /// target. + pub(crate) fn realized_offset_bps(&self) -> f64 { + let public = report::biguint_to_f64(&self.public_best_out); + if public <= 0.0 { + return 0.0; + } + (report::biguint_to_f64(&self.target_out) - public) / public * 10_000.0 + } } impl Injector { @@ -158,6 +228,7 @@ impl Injector { events: solver.market_event_sender(), candidates: Vec::new(), announced: false, + source: None, } } @@ -187,9 +258,12 @@ impl Injector { return Ok(None); }; - let mock = MirrorPool::from_price_pct(source.state, self.config.price_pct); - let price_factor = mock.price_factor(); - let mirrored: Box = Box::new(mock); + // The block's baseline: the mock mirrors the source exactly. Per-order calibration rescales + // it afterwards, but derived data is computed against this state, so an unscaled mirror + // keeps the mock's edge weight in the same range as the real pools it competes + // with. + let mirrored: Box = + Box::new(MirrorPool::from_price_pct(source.state.clone_box(), 100.0)); let first_injection = !self.announced; { let mut state = market.write().await; @@ -230,23 +304,94 @@ impl Injector { ); } - let mock_price = source.price * price_factor; debug!( block, source_component = source.component_id, source_price = source.price, - mock_price, - price_pct = self.config.price_pct, "mirrored source pool onto the mock PropAMM component" ); + let pair_label = source.pair_label.clone(); + self.source = Some(CachedSource { + state: source.state, + token_a: source.token_a, + token_b: source.token_b, + }); Ok(Some(Injected { source_component: source.component_id, source_price: source.price, - mock_price, derived_data_ready, - pair_label: source.pair_label, + pair_label, })) } + + /// Rescales the mock so it quotes essentially nothing, and can therefore never be selected. + /// + /// This is how the harness reads the *public* best route while leaving the graph untouched: the + /// mock's edge stays in place, it just loses every comparison. Cheaper and less disruptive than + /// removing the component, which would mean a topology event and a graph rebuild per order. + pub(crate) async fn neutralize(&self, market: &MarketData) -> bool { + self.rescale(market, MirrorPool::neutral_scale_ppm()) + .await + } + + /// Rescales the mock so its output for this order lands `offset_bps` off `public_best_out`. + /// + /// Returns `None` when the order is off-pair, no source is cached yet, or the source cannot + /// price the order — in each case the caller leaves the order uncalibrated rather than + /// reporting an offset the mock is not actually holding. + /// + /// No derived-data wait here: the topology is unchanged and the scale moves the mock's price by + /// basis points, so its edge weight from the block's baseline injection is still the right + /// order of magnitude for pruning. Waiting per order would cost a recomputation cycle each + /// time. + pub(crate) async fn calibrate( + &self, + market: &MarketData, + token_in: &Address, + amount_in: &BigUint, + public_best_out: &BigUint, + offset_bps: i32, + ) -> Option { + let source = self.source.as_ref()?; + let (from, to) = source.order_tokens(token_in)?; + let target_out = shift_bps(public_best_out, offset_bps); + let scale = MirrorPool::scale_for_target( + source.state.as_ref(), + amount_in.clone(), + from, + to, + &target_out, + )?; + if !self.rescale(market, scale).await { + return None; + } + Some(Calibration { public_best_out: public_best_out.clone(), target_out, offset_bps }) + } + + /// Writes the mock's state at `scale_ppm`. Returns whether a source was cached to rescale. + async fn rescale(&self, market: &MarketData, scale_ppm: u32) -> bool { + let Some(source) = self.source.as_ref() else { + return false; + }; + let scaled: Box = + Box::new(MirrorPool::from_scale_ppm(source.state.clone_box(), scale_ppm)); + market + .write() + .await + .update_states([(MOCK_COMPONENT_ID.to_string(), scaled)]); + true + } +} + +/// Shifts an amount by `offset_bps`, saturating at zero. A negative offset below -10000 bps would +/// otherwise wrap; the mock's price is a ratio and cannot go negative. +fn shift_bps(amount: &BigUint, offset_bps: i32) -> BigUint { + const BPS: i64 = 10_000; + let scaled = i64::from(offset_bps).saturating_add(BPS); + if scaled <= 0 { + return BigUint::ZERO; + } + amount * BigUint::from(scaled.unsigned_abs()) / BigUint::from(BPS.unsigned_abs()) } /// Component ids holding both tokens of the configured pair, excluding the mock itself. @@ -282,6 +427,11 @@ struct BestSource { price: f64, /// The pair as token symbols, e.g. `WETH/USDC`. pair_label: String, + /// First token of the pair, carried so calibration can simulate the source without a market + /// read. + token_a: Token, + /// Second token of the pair. + token_b: Token, } /// The candidate pool the mock mirrors this block. @@ -326,6 +476,8 @@ async fn best_source( state, price, pair_label: format!("{}/{}", token_a.symbol, token_b.symbol), + token_a, + token_b, }) } @@ -426,7 +578,7 @@ mod tests { let config = MirrorConfig { token_a: Bytes::from(vec![0x11; 20]), token_b: Bytes::from(vec![0x22; 20]), - price_pct: 100.05, + offsets_bps: vec![-5, 0, 5], probe_units: 1.0, chain: Chain::Ethereum, }; @@ -446,7 +598,7 @@ mod tests { let config = MirrorConfig { token_a: Bytes::from(vec![0x11; 20]), token_b: Bytes::from(vec![0x22; 20]), - price_pct: 100.0, + offsets_bps: vec![-5, 0, 5], probe_units: 1.0, chain: Chain::Ethereum, }; @@ -458,7 +610,7 @@ mod tests { let config = MirrorConfig { token_a: Bytes::from(vec![0x11; 20]), token_b: Bytes::from(vec![0x22; 20]), - price_pct: 100.0, + offsets_bps: vec![-5, 0, 5], probe_units: 1.0, chain: Chain::Ethereum, }; diff --git a/tools/hindsight/src/propamm/report.rs b/tools/hindsight/src/propamm/report.rs index 33185b90..5843787c 100644 --- a/tools/hindsight/src/propamm/report.rs +++ b/tools/hindsight/src/propamm/report.rs @@ -10,7 +10,10 @@ //! Because the quoted `amount_out` is pinned to the commitment, the mock pool never changes //! hindsight's own win/loss verdict. It only adds this second, orthogonal measurement. -use std::sync::Mutex; +use std::sync::{ + atomic::{AtomicU64, Ordering}, + Mutex, +}; use alloy::primitives::Address; use fynd_core::OrderQuote; @@ -37,6 +40,12 @@ pub(crate) struct Observation { /// Output the mock produced above the commitment — the fee it could have charged. `None` when /// it didn't win. pub fee_headroom: Option, + /// The offset the mock was priced at for this order, in basis points relative to the public + /// best route. `None` for an order that was not calibrated — off the mirrored pair, or + /// unsolvable publicly, so there was no reference to price against. + pub offset_bps: Option, + /// The public best route's output this order's offset was measured against. + pub public_best_out: Option, } impl Observation { @@ -59,9 +68,18 @@ impl Observation { won, committed_amount_out: quote.committed_amount_out().cloned(), fee_headroom: quote.surplus_amount().cloned(), + offset_bps: None, + public_best_out: None, } } + /// Labels this observation with the calibration the mock was priced under. + pub(crate) fn with_calibration(mut self, calibration: &crate::propamm::Calibration) -> Self { + self.offset_bps = Some(calibration.offset_bps); + self.public_best_out = Some(calibration.public_best_out.clone()); + self + } + /// An observation for a solve that never produced a quote. /// /// Recorded so the sink stays index-aligned with the block's trades — the join back to each @@ -74,6 +92,8 @@ impl Observation { won: false, committed_amount_out: None, fee_headroom: None, + offset_bps: None, + public_best_out: None, } } @@ -137,6 +157,10 @@ pub(crate) struct Stats { /// The mirrored pair as token symbols, e.g. `WETH/USDC`. Resolved once the feed has loaded the /// tokens, which is why it is set from the injection path rather than from the CLI. pair_label: Mutex>, + /// Monotonic counter over calibrated orders, which rotates the offset ladder. Counting only + /// the calibrated ones keeps the groups balanced regardless of how much off-pair flow a + /// block has. + order_index: AtomicU64, } impl Stats { @@ -178,6 +202,12 @@ impl Stats { *totals } + /// Hands out the next calibrated order's index, which selects its offset from the ladder. + pub(crate) fn next_order_index(&self) -> u64 { + self.order_index + .fetch_add(1, Ordering::Relaxed) + } + /// Records the mirrored pair's symbol label, so the report can name it. pub(crate) fn set_pair_label(&self, label: &str) { if let Ok(mut pair_label) = self.pair_label.lock() { @@ -222,6 +252,9 @@ pub(crate) fn biguint_to_f64(value: &BigUint) -> f64 { pub(crate) struct Record { /// The mirrored pair as token symbols, e.g. `WETH/USDC` — which pool the mock stood in for. pub pair: Option, + /// The offset the mock was priced at, in basis points relative to the public best route. + /// Absent for an order that was not calibrated. + pub offset_bps: Option, /// Whether the winning route ran through the mock `PropAMM` pool. pub won: bool, /// The public-market output the user is committed to. @@ -246,6 +279,7 @@ impl Record { ) -> Self { Self { pair, + offset_bps: observed.offset_bps, won: observed.won, committed_amount_out: observed .committed_amount_out @@ -273,6 +307,8 @@ mod tests { won, committed_amount_out: won.then(|| BigUint::from(committed)), fee_headroom: won.then(|| BigUint::from(headroom)), + offset_bps: None, + public_best_out: None, } } diff --git a/tools/hindsight/src/report/aggregate.rs b/tools/hindsight/src/report/aggregate.rs index 7c1a5e7b..54bac5e7 100644 --- a/tools/hindsight/src/report/aggregate.rs +++ b/tools/hindsight/src/report/aggregate.rs @@ -10,6 +10,13 @@ use crate::report::record::Comparison; /// Number of order pairs listed in the mock-`PropAMM` breakdown table. const TOP_PROPAMM_PAIRS: usize = 10; +/// Slack allowed when checking a group's fee against its offset, in basis points. +/// +/// The calibration divides integer amounts and the source pool rounds its own output, so a fee +/// lands a fraction of a bps off its target. Anything beyond this is a real discrepancy, not +/// rounding. +const FEE_TOLERANCE_BPS: f64 = 0.5; + /// Number of trades listed in the biggest-wins and biggest-losses tables. const TOP_TRADES: usize = 10; /// Number of tokens listed in the unsolvable-tail table. @@ -57,6 +64,67 @@ pub(crate) struct PropAmm { pub median_headroom_bps: Option, /// Per-order-pair breakdown, most wins first. pub by_pair: Vec, + /// Per-offset groups, ascending. Empty for a run with no calibrated orders. + pub groups: Vec, +} + +/// Whether an offset group behaved as its price implies. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum GroupVerdict { + /// The group met its expectation. + Pass, + /// The group violated it; the string says how. + Fail(String), + /// No calibrated order landed in this group, so there is nothing to conclude. + NoData, +} + +/// One offset group: orders priced the same distance from the public best route, and whether the +/// router treated them the way that price implies. +/// +/// This is the harness's actual test. Each order's competitive situation is constructed, so the +/// group is an assertion rather than a measurement: +/// +/// - **below the market** (`offset_bps < 0`) — the mock is strictly worse, so it must never be +/// selected; +/// - **at the market** (`offset_bps == 0`) — it can only win on gas, and then there is no surplus, +/// so any selection must carry a zero fee; +/// - **above the market** (`offset_bps > 0`) — the fee taken cannot exceed the offset, because the +/// offset is all the surplus there is. +pub(crate) struct PropAmmGroup { + /// The offset these orders were priced at, in basis points off the public best route. + pub offset_bps: i32, + /// Calibrated orders in this group. + pub orders: usize, + /// Of those, how many the router routed through the mock. + pub selected: usize, + /// Largest fee taken in the group, in bps — the number the expectation is checked against. + pub max_fee_bps: Option, + /// Median fee taken, in bps. + pub median_fee_bps: Option, + /// Whether the group met its expectation. + pub verdict: GroupVerdict, +} + +impl PropAmmGroup { + /// The expectation this group's price implies, as a short phrase for the report. + pub(crate) fn expectation(&self) -> &'static str { + match self.offset_bps.cmp(&0) { + std::cmp::Ordering::Less => "never selected", + std::cmp::Ordering::Equal => "selected only on gas, zero fee", + std::cmp::Ordering::Greater => "selected, fee at most the offset", + } + } + + /// Share of the group's orders the mock was selected for, as a percentage. + // Precision loss is irrelevant: these are order counts, far below f64's exact-integer range. + #[expect(clippy::cast_precision_loss)] + pub(crate) fn selected_pct(&self) -> f64 { + if self.orders == 0 { + return 0.0; + } + self.selected as f64 / self.orders as f64 * 100.0 + } } impl PropAmm { @@ -196,9 +264,106 @@ fn propamm(records: &[Comparison]) -> Option { fee_headroom_usd: sum_propamm(&wins, |p| p.fee_headroom_usd), median_headroom_bps: median(&mut headroom_bps), by_pair: propamm_by_pair(&scoped), + groups: propamm_groups(&scoped), }) } +/// Groups the calibrated orders by offset and judges each group against its price. +/// +/// Orders with no offset are excluded: they were not calibrated, so no expectation applies to them. +fn propamm_groups(scoped: &[&Comparison]) -> Vec { + let mut grouped: HashMap> = HashMap::new(); + for record in scoped { + if let Some(offset) = record + .propamm + .as_ref() + .and_then(|p| p.offset_bps) + { + grouped + .entry(offset) + .or_default() + .push(record); + } + } + + let mut groups: Vec = grouped + .into_iter() + .map(|(offset_bps, records)| { + let mut fees: Vec = records + .iter() + .filter(|r| { + r.propamm + .as_ref() + .is_some_and(|p| p.won) + }) + .filter_map(|r| { + r.propamm + .as_ref() + .and_then(|p| p.fee_headroom_bps) + }) + .collect(); + let selected = records + .iter() + .filter(|r| { + r.propamm + .as_ref() + .is_some_and(|p| p.won) + }) + .count(); + let max_fee_bps = fees + .iter() + .copied() + .fold(None::, |acc, fee| Some(acc.map_or(fee, |best| best.max(fee)))); + let mut group = PropAmmGroup { + offset_bps, + orders: records.len(), + selected, + max_fee_bps, + median_fee_bps: median(&mut fees), + verdict: GroupVerdict::NoData, + }; + group.verdict = judge_group(&group); + group + }) + .collect(); + groups.sort_by_key(|group| group.offset_bps); + groups +} + +/// Judges one offset group against the expectation its price implies. +fn judge_group(group: &PropAmmGroup) -> GroupVerdict { + if group.orders == 0 { + return GroupVerdict::NoData; + } + if group.offset_bps < 0 { + return if group.selected == 0 { + GroupVerdict::Pass + } else { + GroupVerdict::Fail(format!( + "priced {} bps below the public market, yet selected for {} of {} orders", + -group.offset_bps, group.selected, group.orders + )) + }; + } + if group.selected == 0 { + // Not a failure at zero offset — the mock only wins there on gas, and it need not be + // cheaper. Above zero it should win, so say so without calling the run broken. + return GroupVerdict::NoData; + } + let Some(max_fee) = group.max_fee_bps else { + return GroupVerdict::NoData; + }; + let ceiling = f64::from(group.offset_bps) + FEE_TOLERANCE_BPS; + if max_fee <= ceiling { + GroupVerdict::Pass + } else { + GroupVerdict::Fail(format!( + "priced {} bps above the public market, yet took a fee of {max_fee:.2} bps", + group.offset_bps + )) + } +} + /// Sums an optional USD field over records, skipping the ones where the token was not priced. fn sum_propamm( records: &[&Comparison], @@ -651,4 +816,135 @@ mod tests { assert_eq!(rows[1].won, 0); assert!(rows[1].median_headroom_bps.is_none()); } + + /// A calibrated record: priced `offset_bps` off the public best route, selected or not. + fn calibrated(block: u64, offset_bps: i32, won: bool, fee_bps: f64) -> Comparison { + serde_json::from_value(serde_json::json!({ + "block": block, + "settled_tx": format!("0x{block:064x}"), + "venue": "relay", + "solver": "1inch", + "token_in": "0xweth", + "token_out": "0xusdt", + "top": { "verdict": "win", "net_bps": 5.0, "settled_value_usd": 1000.0 }, + "propamm": { + "pair": "WETH/USDT", + "offset_bps": offset_bps, + "won": won, + "fee_headroom_bps": won.then_some(fee_bps), + "committed_usd": won.then_some(1_000.0), + "fee_headroom_usd": won.then(|| fee_bps / 10_000.0 * 1_000.0), + }, + })) + .unwrap() + } + + #[test] + fn test_groups_are_ordered_by_price_ascending() { + let records = vec![ + calibrated(1, 5, true, 5.0), + calibrated(2, -5, false, 0.0), + calibrated(3, 0, true, 0.0), + ]; + let groups = propamm(&records) + .expect("outcomes present") + .groups; + let offsets: Vec = groups + .iter() + .map(|g| g.offset_bps) + .collect(); + assert_eq!(offsets, vec![-5, 0, 5], "cards read below-market to above-market"); + } + + #[test] + fn test_below_market_group_passes_only_when_never_selected() { + let never = vec![calibrated(1, -5, false, 0.0), calibrated(2, -5, false, 0.0)]; + assert_eq!(propamm(&never).unwrap().groups[0].verdict, GroupVerdict::Pass); + + // A single selection below the public market breaks the router's core promise, so the group + // must fail loudly rather than average the violation away. + let once = vec![calibrated(1, -5, false, 0.0), calibrated(2, -5, true, 3.0)]; + let verdict = &propamm(&once).unwrap().groups[0].verdict; + assert!( + matches!(verdict, GroupVerdict::Fail(reason) if reason.contains("below the public")), + "expected a failure naming the violation, got {verdict:?}" + ); + } + + #[test] + fn test_at_market_group_passes_on_a_zero_fee() { + // At the market the mock can only win on gas, and then there is no surplus to take. + let records = vec![calibrated(1, 0, true, 0.0), calibrated(2, 0, false, 0.0)]; + let group = &propamm(&records).unwrap().groups[0]; + assert_eq!(group.verdict, GroupVerdict::Pass); + assert_eq!(group.selected, 1); + } + + #[test] + fn test_at_market_group_fails_on_a_non_zero_fee() { + // Charging a fee at the market price means the user was quoted below the public route. + let records = vec![calibrated(1, 0, true, 4.0)]; + assert!(matches!(&propamm(&records).unwrap().groups[0].verdict, GroupVerdict::Fail(_))); + } + + #[test] + fn test_above_market_group_caps_the_fee_at_the_offset() { + // The offset is all the surplus there is, so a fee within it passes... + let within = vec![calibrated(1, 5, true, 4.8), calibrated(2, 5, true, 5.0)]; + assert_eq!(propamm(&within).unwrap().groups[0].verdict, GroupVerdict::Pass); + + // ...and a fee beyond it means the pool signed more than it could charge. + let beyond = vec![calibrated(1, 5, true, 5.0), calibrated(2, 5, true, 9.0)]; + let verdict = &propamm(&beyond).unwrap().groups[0].verdict; + assert!( + matches!(verdict, GroupVerdict::Fail(reason) if reason.contains("9.00")), + "expected the offending fee in the message, got {verdict:?}" + ); + } + + #[test] + fn test_above_market_fee_tolerance_absorbs_rounding_only() { + // Integer division puts a fee a hair over its target; that is rounding, not a discrepancy. + let rounding = vec![calibrated(1, 5, true, 5.0 + FEE_TOLERANCE_BPS / 2.0)]; + assert_eq!(propamm(&rounding).unwrap().groups[0].verdict, GroupVerdict::Pass); + + let real = vec![calibrated(1, 5, true, 5.0 + FEE_TOLERANCE_BPS * 4.0)]; + assert!(matches!(&propamm(&real).unwrap().groups[0].verdict, GroupVerdict::Fail(_))); + } + + #[test] + fn test_group_with_no_selection_above_market_is_no_data_not_failure() { + // Winning above the market depends on gas as well as price, so never winning is + // inconclusive rather than a violation — the harness must not cry wolf. + let records = vec![calibrated(1, 5, false, 0.0)]; + assert_eq!(propamm(&records).unwrap().groups[0].verdict, GroupVerdict::NoData); + } + + #[test] + fn test_uncalibrated_orders_form_no_group() { + // Off-pair orders carry no offset, so no expectation applies and they must not dilute a + // group. + let records = vec![ + propamm_record(1, "0xdai", "0xusdc", false, 0.0, 0.0), + calibrated(2, 5, true, 5.0), + ]; + let propamm = propamm(&records).expect("outcomes present"); + assert_eq!(propamm.groups.len(), 1); + assert_eq!(propamm.groups[0].orders, 1); + // Both still count toward the run's solved total. + assert_eq!(propamm.solved, 2); + } + + #[test] + fn test_selected_pct_is_zero_rather_than_nan_when_empty() { + let group = PropAmmGroup { + offset_bps: 0, + orders: 0, + selected: 0, + max_fee_bps: None, + median_fee_bps: None, + verdict: GroupVerdict::NoData, + }; + assert!(group.selected_pct().abs() < f64::EPSILON); + } } diff --git a/tools/hindsight/src/report/html.rs b/tools/hindsight/src/report/html.rs index 8842cf6a..edd8862c 100644 --- a/tools/hindsight/src/report/html.rs +++ b/tools/hindsight/src/report/html.rs @@ -9,7 +9,8 @@ use std::fmt::Write as _; use crate::report::aggregate::{ - Count, GroupStats, PropAmm, PropAmmPair, Report, Savings, Summary, TradeRow, VerdictStat, + Count, GroupStats, GroupVerdict, PropAmm, PropAmmGroup, PropAmmPair, Report, Savings, Summary, + TradeRow, VerdictStat, }; /// Shortest share of a stacked column that gets an inline `12.3%` label. Below it the segment is @@ -298,16 +299,80 @@ fn propamm_section(propamm: &PropAmm) -> String { stat(&fmt_bps(propamm.avg_headroom_bps()), "fee headroom bps (flow-weighted)"), ); let body = format!( - "

Mirrored pool: {}, priced fee-free. \ - Fee headroom is what the signed extension could have charged and still beaten the \ - public market — measured, not assumed. Quotes from this pool are not executable.

\ - {headline}{}", + "

Mirrored pool: {}, priced fee-free against the public \ + best route for each order. Fee headroom is what the signed extension could have \ + charged and still beaten that route — measured, not assumed. Quotes from this pool are not \ + executable.

{}{headline}{}", escape(pair), + propamm_groups(&propamm.groups), propamm_pair_table(&propamm.by_pair), ); section("Mock PropAMM (exclusive route)", &body) } +/// The offset groups, one card each, ascending by price. +/// +/// Each card leads with its verdict rather than its numbers: the whole point of calibrating every +/// order is that the expected behaviour is known up front, so the reader should not have to work +/// out whether 5.2% is good. The numbers are underneath for when it fails. +fn propamm_groups(groups: &[PropAmmGroup]) -> String { + if groups.is_empty() { + return "

No calibrated orders — no settled trade in this run was on \ + the mirrored pair.

" + .to_string(); + } + let mut cards = String::new(); + for group in groups { + let (chip, cls) = match &group.verdict { + GroupVerdict::Pass => ("pass".to_string(), "ok"), + GroupVerdict::Fail(_) => ("FAIL".to_string(), "bad"), + GroupVerdict::NoData => ("no data".to_string(), "idle"), + }; + let detail = match &group.verdict { + GroupVerdict::Fail(reason) => { + format!("

{}

", escape(reason)) + } + GroupVerdict::Pass | GroupVerdict::NoData => String::new(), + }; + let _ = write!( + cards, + "
\ +
{}\ + {chip}
\ +
{}
\ + \ + \ + \ + \ + \ +
orders{}
selected{} ({})
fee taken (median){}
fee taken (max){}
{detail}\ +
", + escape(&fmt_offset(group.offset_bps)), + escape(group.expectation()), + group.orders, + group.selected, + share_pct_of(group.selected_pct()), + fmt_bps(group.median_fee_bps), + fmt_bps(group.max_fee_bps), + ); + } + format!("
{cards}
") +} + +/// An offset as a signed bps label, so a column header reads as a price and not a bare number. +fn fmt_offset(offset_bps: i32) -> String { + match offset_bps.cmp(&0) { + std::cmp::Ordering::Less => format!("{offset_bps} bps (below market)"), + std::cmp::Ordering::Equal => "at market".to_string(), + std::cmp::Ordering::Greater => format!("+{offset_bps} bps (above market)"), + } +} + +/// A percentage that is already computed, rendered like the report's other shares. +fn share_pct_of(pct: f64) -> String { + format!("{pct:.0}%") +} + /// Per-order-direction breakdown. A row's pair can differ from the mirrored pair: the mock serves /// any route with a leg it can price, so a `DAI→USDC` order routed through `WETH` shows up here /// too. @@ -476,6 +541,19 @@ section { background: #211a30; border: 1px solid #362b4a; border-radius: 8px; .nodata { color: #9a8bbf; margin: 0; } /* A section's framing sentence: what the numbers below mean, before they are read. */ .note { color: #b9adcf; margin: 0 0 .5rem; max-width: 68ch; line-height: 1.55; } +/* One card per offset group. They wrap rather than scroll, so a wider ladder stays readable. */ +.groups { display: flex; flex-wrap: wrap; gap: 1rem; margin: 1.25rem 0; } +.group { flex: 1 1 15rem; border: 1px solid #2f2540; border-radius: 6px; padding: .9rem 1rem; } +.grouphead { display: flex; align-items: center; justify-content: space-between; gap: .75rem; } +.groupoff { font-weight: 600; } +.groupexp { color: #9a8bbf; font-size: .75rem; margin-top: .2rem; min-height: 2.2em; } +.groupnums { margin-top: .6rem; width: 100%; } +.groupnums td { padding: .15rem 0; border: 0; } +/* The verdict never rests on colour alone: each chip is also labelled pass / FAIL / no data. */ +.chip.ok { background: #1b3a1f; color: #a5d6a7; } +.chip.bad { background: #4a1c1c; color: #ef9a9a; } +.chip.idle { background: #2f2540; color: #9a8bbf; } +.groupfail { color: #ef9a9a; font-size: .78rem; margin: .5rem 0 0; line-height: 1.45; } table { border-collapse: collapse; width: 100%; } th, td { text-align: left; padding: .4rem .6rem; border-bottom: 1px solid #362b4a; } th { color: #9a8bbf; font-weight: 600; font-size: .8rem; text-transform: uppercase; } @@ -677,4 +755,63 @@ mod tests { assert!(html.contains("Mock PropAMM (exclusive route)")); assert!(html.contains("0.0%"), "a zero winrate must be stated, not omitted"); } + + /// A report from calibrated records across the three offset groups. + fn group_report(above_fee_bps: f64) -> Report { + let record = |block: u64, offset: i32, won: bool, fee: f64| { + serde_json::json!({ + "block": block, + "settled_tx": format!("0x{block:064x}"), + "venue": "relay", "solver": "1inch", + "token_in": "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2", + "token_out": "0xdac17f958d2ee523a2206206994597c13d831ec7", + "top": {"verdict": "win", "net_bps": 5.0, "settled_value_usd": 1000.0}, + "propamm": { + "pair": "WETH/USDT", "offset_bps": offset, "won": won, + "fee_headroom_bps": won.then_some(fee), + "committed_usd": won.then_some(1_000.0), + "fee_headroom_usd": won.then_some(fee / 10.0), + }, + }) + }; + let records: Vec = vec![ + record(1, -5, false, 0.0), + record(2, 0, true, 0.0), + record(3, 5, true, above_fee_bps), + ] + .into_iter() + .map(|v| serde_json::from_value(v).unwrap()) + .collect(); + build(&records) + } + + #[test] + fn test_group_cards_lead_with_the_verdict_and_name_each_price() { + let html = render(&group_report(5.0), None); + assert!(html.contains("-5 bps (below market)")); + assert!(html.contains("at market")); + assert!(html.contains("+5 bps (above market)")); + // The verdict is a word, never colour alone. + assert!(html.contains(">pass<"), "a passing group must say so in text"); + assert!(html.contains("never selected")); + assert!(html.contains("selected only on gas, zero fee")); + } + + #[test] + fn test_a_failing_group_renders_the_reason_not_just_a_colour() { + let html = render(&group_report(20.0), None); + assert!(html.contains(">FAIL<")); + assert!( + html.contains("took a fee of 20.00 bps"), + "the reason must be readable without opening the JSONL" + ); + } + + #[test] + fn test_group_panel_says_so_when_nothing_was_calibrated() { + // A run whose settled trades never touched the mirrored pair must explain the empty panel + // rather than render three blank cards. + let html = render(&propamm_report(true), None); + assert!(html.contains("No calibrated orders")); + } } diff --git a/tools/hindsight/src/report/record.rs b/tools/hindsight/src/report/record.rs index fe2225e5..2dc80e9e 100644 --- a/tools/hindsight/src/report/record.rs +++ b/tools/hindsight/src/report/record.rs @@ -34,6 +34,11 @@ pub(crate) struct PropAmm { /// The mirrored pair as token symbols, e.g. `WETH/USDC`. #[serde(default)] pub pair: Option, + /// The offset the mock was priced at, in basis points relative to the public best route for + /// this order. Absent for an order that was not calibrated, which is excluded from the + /// group test. + #[serde(default)] + pub offset_bps: Option, /// Whether the winning route ran through the mock pool. pub won: bool, /// That headroom as a fraction of the committed output, in basis points. @@ -161,6 +166,8 @@ mod tests { won: true, committed_amount_out: Some(1_000_000_000u64.into()), fee_headroom: Some(400_000u64.into()), + offset_bps: Some(5), + public_best_out: Some(1_000_000_000u64.into()), }; let record = Record::new(&observed, Some("WETH/USDC".to_string()), Some(1_000.0), Some(0.4)); @@ -182,6 +189,7 @@ mod tests { .propamm .expect("the propamm field round-trips"); assert_eq!(propamm.pair.as_deref(), Some("WETH/USDC")); + assert_eq!(propamm.offset_bps, Some(5)); assert!(propamm.won); // 400_000 / 1_000_000_000 = 4 bps. assert!((propamm.fee_headroom_bps.unwrap() - 4.0).abs() < 1e-9); diff --git a/tools/hindsight/src/resolve/monitor.rs b/tools/hindsight/src/resolve/monitor.rs index 1eadd538..b97a6fda 100644 --- a/tools/hindsight/src/resolve/monitor.rs +++ b/tools/hindsight/src/resolve/monitor.rs @@ -128,12 +128,25 @@ pub(crate) struct MonitorArgs { #[arg(long, env = "PROPAMM_PAIR", value_delimiter = ',')] pub propamm_pair: Option>, - /// The mock pool's fee-free price, as a percentage of the best real pool's price for the pair. - /// `100` positions it exactly at the best price we can see (the control case — it cannot then - /// strictly beat the public market); `100.05` positions it 5 bps better. The fee it could - /// charge on top is measured, not set here - #[arg(long, env = "PROPAMM_PRICE_PCT", default_value_t = 100.0)] - pub propamm_price_pct: f64, + /// Price offsets, in basis points relative to **the public best route's output for the order + /// being solved**. Orders on the mirrored pair cycle through this list, so one run fills every + /// group and each order becomes an assertion rather than a data point: + /// + /// - a negative offset prices the mock below the public market and must never be selected; + /// - `0` matches it exactly, so any selection is on gas alone and must carry a zero fee; + /// - a positive offset is the underbid, and the fee taken must not exceed it. + /// + /// The mock's fee-free price is set per order, so "5 bps better" means 5 bps better than the + /// route Fynd would otherwise have quoted — not 5 bps better than some single pool + // `allow_hyphen_values` so a negative offset reads as a value, not as a short flag. + #[arg( + long, + env = "PROPAMM_OFFSETS_BPS", + value_delimiter = ',', + allow_hyphen_values = true, + default_values_t = [-5, 0, 5] + )] + pub propamm_offsets_bps: Vec, /// Trade size, in whole units of the pair's first token, used each block to pick which real /// pool to mirror. Set it near the sizes being re-solved so the mirror tracks the pool that @@ -149,6 +162,16 @@ pub(crate) struct MonitorArgs { struct PropAmmHarness { injector: tokio::sync::Mutex, stats: Arc, + /// The pair and offset ladder, kept outside the injector's mutex so the solve path can decide + /// whether an order is calibratable without taking that lock. + config: propamm::MirrorConfig, +} + +impl PropAmmHarness { + /// The mirrored pair and offset ladder. + fn config(&self) -> &propamm::MirrorConfig { + &self.config + } } /// Drives the in-process solver, stepping the chain one block per `SteppingSolver::advance`. @@ -244,7 +267,6 @@ impl StepAdapter<'_> { block, source = injected.source_component, source_price = injected.source_price, - mock_price = injected.mock_price, derived_data_ready = injected.derived_data_ready, "mirrored the mock PropAMM pool" ); @@ -305,12 +327,113 @@ impl StepAdapter<'_> { } } -#[async_trait] -impl SteppingSolver for StepAdapter<'_> { - async fn solve(&self, token_in: Address, token_out: Address, amount_in: U256) -> Outcome { +impl StepAdapter<'_> { + /// Solve an order twice so the mock is priced against what Fynd would otherwise have quoted. + /// + /// Pass one neutralises the mock, so the quote is the public best route. Pass two rescales the + /// mock to land `offset_bps` off that output and re-quotes, and its result is the one returned + /// — the order's real answer, with a known-by-construction competitive situation. + /// + /// Falls back to the pass-one result whenever calibration is impossible (no source cached yet, + /// or the source cannot price the order). The order then carries no offset and is reported + /// as an ordinary comparison rather than as a group member. + async fn solve_calibrated( + &self, + harness: &PropAmmHarness, + token_in: Address, + token_out: Address, + amount_in: U256, + offset_bps: i32, + ) -> (Outcome, Option) { + let market = harness_market(self.solver); + let injector = harness.injector.lock().await; + + if !injector.neutralize(&market).await { + return self + .quote_order(token_in, token_out, amount_in) + .await; + } + let (public_outcome, public_observed) = self + .quote_order(token_in, token_out, amount_in) + .await; + let Outcome::Solved(public) = &public_outcome else { + // Fynd cannot serve this order publicly, so there is no reference to underbid. Report + // the unsolvable outcome as-is rather than inventing a price for the mock + // to beat. + return (public_outcome, public_observed); + }; + let Ok(public_best_out) = public + .amount_out + .to_string() + .parse::() + else { + return (public_outcome, public_observed); + }; + + let core_token_in = CoreAddress::from(token_in.into_array()); + let Some(calibration) = injector + .calibrate( + &market, + &core_token_in, + &amount_in_biguint(amount_in), + &public_best_out, + offset_bps, + ) + .await + else { + return (public_outcome, public_observed); + }; + drop(injector); + + // The calibration's own error check: integer division and the source pool's rounding mean + // the target lands a fraction of a bps off the requested offset. A large gap means + // the source could not hold the target, which would make the group's expectation + // untestable. + let realized_bps = calibration.realized_offset_bps(); + if (realized_bps - f64::from(offset_bps)).abs() > 0.5 { + warn!( + offset_bps, + realized_bps, + "mock PropAMM calibration missed its target offset; this order's group result is \ + not trustworthy" + ); + } + debug!( + offset_bps, + realized_bps, "calibrated the mock PropAMM against the public best route" + ); + let (outcome, observed) = self .quote_order(token_in, token_out, amount_in) .await; + let observed = observed.map(|o| o.with_calibration(&calibration)); + (outcome, observed) + } +} + +#[async_trait] +impl SteppingSolver for StepAdapter<'_> { + async fn solve(&self, token_in: Address, token_out: Address, amount_in: U256) -> Outcome { + let calibrated_offset = self.propamm.and_then(|harness| { + let config = harness.config(); + let core_in = CoreAddress::from(token_in.into_array()); + let core_out = CoreAddress::from(token_out.into_array()); + config + .serves(&core_in, &core_out) + .then(|| config.offset_for(harness.stats.next_order_index())) + .flatten() + }); + + let (outcome, observed) = match (self.propamm, calibrated_offset) { + (Some(harness), Some(offset_bps)) => { + self.solve_calibrated(harness, token_in, token_out, amount_in, offset_bps) + .await + } + _ => { + self.quote_order(token_in, token_out, amount_in) + .await + } + }; // Recorded on every path, including the ones that never produced a quote: the sink is // joined back to the block's trades by position, so a skipped solve would shift // every later observation onto the wrong trade. `Observation::solved` is what the @@ -614,8 +737,9 @@ pub(crate) async fn run(cfg: MonitorArgs) -> anyhow::Result<()> { let propamm = propamm_config .clone() .map(|config| PropAmmHarness { - injector: tokio::sync::Mutex::new(propamm::Injector::new(&solver, config)), + injector: tokio::sync::Mutex::new(propamm::Injector::new(&solver, config.clone())), stats: Arc::clone(&propamm_stats), + config, }); let adapter = StepAdapter { solver: &solver, @@ -688,14 +812,14 @@ fn propamm_config( let config = propamm::MirrorConfig { token_a, token_b, - price_pct: cfg.propamm_price_pct, + offsets_bps: cfg.propamm_offsets_bps.clone(), probe_units: cfg.propamm_probe_units, chain, }; info!( token_a = %config.token_a, token_b = %config.token_b, - price_pct = config.price_pct, + offsets_bps = ?config.offsets_bps, probe_units = config.probe_units, "mock PropAMM enabled; its quotes are not executable" ); @@ -706,7 +830,7 @@ fn propamm_config( /// headroom that came out. fn log_propamm_summary(config: &propamm::MirrorConfig, totals: &propamm::report::Totals) { info!( - price_pct = config.price_pct, + offsets_bps = ?config.offsets_bps, solved_orders = totals.solved, propamm_wins = totals.won, winrate_pct = format!("{:.1}", totals.winrate_pct()), @@ -800,6 +924,16 @@ fn report_propamm_block( vec![None; trade_count] } +/// The solver's market-data handle, which the injector writes the mock's state through. +fn harness_market(solver: &Solver) -> fynd_core::feed::market_data::MarketData { + solver.market_data() +} + +/// Converts a `U256` order size to `BigUint`, which is what the mock's calibration math uses. +fn amount_in_biguint(amount_in: U256) -> BigUint { + BigUint::from_bytes_be(&amount_in.to_be_bytes::<32>()) +} + /// Converts a `BigUint` to `U256`, or `None` when it does not fit — the amount is then left /// unvalued rather than silently reported as zero. fn biguint_to_u256_opt(value: &BigUint) -> Option { @@ -1108,7 +1242,7 @@ mod tests { max_lag_blocks: Some(100), comparisons_dir: None, propamm_pair: None, - propamm_price_pct: 100.05, + propamm_offsets_bps: vec![-5, 0, 5], propamm_probe_units: 1.0, }) .await From 7e6636f976ce5bb6f9e8261d4c8144710959fb22 Mon Sep 17 00:00:00 2001 From: TAMARA LIPOWSKI Date: Tue, 28 Jul 2026 16:29:55 -0400 Subject: [PATCH 08/12] docs(hindsight): describe the calibrated offset groups --- tools/hindsight/CLAUDE.md | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/tools/hindsight/CLAUDE.md b/tools/hindsight/CLAUDE.md index 9174260d..2816d973 100644 --- a/tools/hindsight/CLAUDE.md +++ b/tools/hindsight/CLAUDE.md @@ -47,7 +47,7 @@ and takes neither. | `ALLIUM_QUERY_ID` | Saved Allium query ID (`verify` only) | | `HINDSIGHT_REGISTRY` | Override path for the decoder address-book TOML | | `PROPAMM_PAIR` | Token pair the mock PropAMM mirrors, comma-separated (`monitor` only) | -| `PROPAMM_PRICE_PCT` | Mock PropAMM's fee-free price as a percentage of the best real pool's | +| `PROPAMM_OFFSETS_BPS` | Price offsets in bps off the public best route, e.g. `-5,0,5` | | `PROPAMM_PROBE_UNITS` | Trade size used to pick which real pool the mock mirrors | ## Architecture @@ -118,14 +118,24 @@ pool exists. Off unless `monitor --propamm-pair` is set. | `mirror.rs` | `MirrorPool` — a `ProtocolSim` that delegates to the best real pool for the pair and scales its price by `--propamm-price-pct`, charging no fee | | `report.rs` | Per-order outcomes and run totals; `Record` is what lands in the comparisons JSONL | -Two knobs, deliberately separate: +Each order on the mirrored pair is solved **twice**: once with the mock neutralised (scaled to one +part per million, so it stays in the graph but loses every comparison), which yields the public best +route Fynd would otherwise have quoted; then again with the mock rescaled so its output lands +`--propamm-offsets-bps` off that number. So an offset means "this much better than the route Fynd +would have quoted", not "this much better than some single pool". -- **Input** — `--propamm-price-pct` is the pool's fee-free price as a percentage of the best real - pool's price for the pair. `100` places it exactly at the best price we can see (the control case: - the router requires a *strict* beat, so it must never win); `100.05` places it 5 bps better. -- **Output** — *fee headroom*: the mock charges nothing, so whatever the router finds above the - public commitment is the fee the signed extension could have charged and still won the trade. - Reported per trade, per order pair, and as a flow-weighted run average. +That makes every calibrated order an assertion. The report groups them by offset and judges each +group against the behaviour its price implies: + +| offset | expectation | +|---|---| +| below market | never selected — the router requires a strict beat | +| at market | can only win on gas, and then there is no surplus, so the fee must be zero | +| above market | the fee taken cannot exceed the offset, since the offset is all the surplus there is | + +A group with no selections above market is reported as *no data*, not a failure: winning there +depends on gas as well as price. Orders off the mirrored pair carry no offset and form no group — +calibration is only exact when the mock serves the whole order in one hop. Fynd's existing exclusive-access routing does the rest: `FyndBuilder::exclusivity_policy` hides the mock from every configured worker pool, and each pool is twinned with a `liquidity_scope = "all"` @@ -140,10 +150,12 @@ Two things to get right when running it: - **Set `EXCLUSIVE_SWAP_CONTROLLER_KEY`** (any throwaway key — nothing is executed). The encoder fails fast on an exclusive leg with no signer, which turns every win into a failed quote and silently reports a zero winrate. -- **Pick a pair that has flow, and check the routes.** The exclusive leg must be terminal in its - path and within the pool's hop budget, so a pair absent from the block's settled trades yields - zero wins for reasons that have nothing to do with the price. Sanity-check with a deliberately - absurd `--propamm-price-pct 105`: the measured `fee_headroom_bps` should come back at ~500. +- **Pick a pair that has flow.** Only orders whose own pair is the mirrored one get calibrated, and + those are a small slice of settled flow — 40 mainnet blocks yielded 9 calibrated orders on + ETH/USDT. On ethereum, ETH/USDC carries roughly twice ETH/USDT's volume; check a run's per-pair + table before committing to a long one. Filling the groups is a matter of blocks, so budget for it. +- **`--min-tvl 10`** is mandatory against `tycho-fynd-ethereum`, which rejects any other value with + `tvl_gt must be == 10`. ### Verdict model From 98d69de2b2aa8c2b982d0ec1c7b4fb6cc090e825 Mon Sep 17 00:00:00 2001 From: TAMARA LIPOWSKI Date: Tue, 28 Jul 2026 20:48:15 -0400 Subject: [PATCH 09/12] feat(hindsight): compare Fynd with and without the mock PropAMM The two-pass calibration already solves both worlds, so keeping the neutralised pass's amounts turns every calibrated order into a controlled A/B: same order, same block state, the mock's presence the only difference. The report shows win rate, quoted output and captured fee side by side, plus a verdict per offset group rather than only overall. --- tools/hindsight/src/propamm/mod.rs | 18 +- tools/hindsight/src/propamm/report.rs | 53 ++++- tools/hindsight/src/report/aggregate.rs | 244 +++++++++++++++++++ tools/hindsight/src/report/html.rs | 296 +++++++++++++++++++----- tools/hindsight/src/report/record.rs | 30 ++- tools/hindsight/src/resolve/monitor.rs | 154 +++++++++++- 6 files changed, 728 insertions(+), 67 deletions(-) diff --git a/tools/hindsight/src/propamm/mod.rs b/tools/hindsight/src/propamm/mod.rs index 87b994df..3b1ef5e1 100644 --- a/tools/hindsight/src/propamm/mod.rs +++ b/tools/hindsight/src/propamm/mod.rs @@ -475,7 +475,7 @@ async fn best_source( component_id, state, price, - pair_label: format!("{}/{}", token_a.symbol, token_b.symbol), + pair_label: format!("{}/{}", token_symbol(&token_a), token_symbol(&token_b)), token_a, token_b, }) @@ -542,6 +542,22 @@ pub(crate) fn is_mock_component(component: &ProtocolComponent) -> bool { component.id == MOCK_COMPONENT_ID } +/// A token's display symbol. +/// +/// The zero address is native ETH by the convention this codebase already uses for it (the +/// decoder's `native_wrapper` hops, the `wrapped_native` registry entry). It carries no symbol of +/// its own in the token registry, where it would otherwise render as forty hex characters. +fn token_symbol(token: &Token) -> String { + if token + .address + .iter() + .all(|byte| *byte == 0) + { + return "ETH".to_string(); + } + token.symbol.clone() +} + /// Formats a token amount in whole units, for logging. pub(crate) fn to_units(amount: &BigUint, token: &Token) -> f64 { report::biguint_to_f64(amount) / 10f64.powi(decimals_exponent(token)) diff --git a/tools/hindsight/src/propamm/report.rs b/tools/hindsight/src/propamm/report.rs index 5843787c..972a96cc 100644 --- a/tools/hindsight/src/propamm/report.rs +++ b/tools/hindsight/src/propamm/report.rs @@ -46,6 +46,21 @@ pub(crate) struct Observation { pub offset_bps: Option, /// The public best route's output this order's offset was measured against. pub public_best_out: Option, + /// The same order's result with the mock neutralised — the "without `PropAMM`" world. + /// + /// The two-pass calibration already solves that world to find the reference price, so keeping + /// its amounts turns every calibrated order into a controlled A/B: same block, same state, + /// same order, the mock's presence the only difference. + pub without: Option, +} + +/// One order's result with the mock neutralised. +#[derive(Debug, Clone)] +pub(crate) struct PublicOnly { + /// Output of the best public-only route. + pub amount_out: BigUint, + /// That output net of the route's gas. + pub amount_out_net_gas: BigUint, } impl Observation { @@ -70,13 +85,20 @@ impl Observation { fee_headroom: quote.surplus_amount().cloned(), offset_bps: None, public_best_out: None, + without: None, } } - /// Labels this observation with the calibration the mock was priced under. - pub(crate) fn with_calibration(mut self, calibration: &crate::propamm::Calibration) -> Self { + /// Labels this observation with the calibration the mock was priced under, and with the + /// public-only result the offset was measured against. + pub(crate) fn with_calibration( + mut self, + calibration: &crate::propamm::Calibration, + without: PublicOnly, + ) -> Self { self.offset_bps = Some(calibration.offset_bps); self.public_best_out = Some(calibration.public_best_out.clone()); + self.without = Some(without); self } @@ -94,6 +116,7 @@ impl Observation { fee_headroom: None, offset_bps: None, public_best_out: None, + without: None, } } @@ -234,6 +257,18 @@ impl Stats { } } +/// One order scored in both worlds, as the caller computed it against the settled trade. +/// +/// Every field is optional because a world can be unscoreable independently: the order may have +/// been uncalibrated (no "without" pass at all), or its output token unpriced (amounts but no USD). +#[derive(Debug, Default, Clone, Copy)] +pub(crate) struct AbResult { + pub without_won: Option, + pub with_won: Option, + pub without_improvement_usd: Option, + pub with_improvement_usd: Option, +} + /// Converts a `BigUint` to `f64` for ratio reporting. Saturates to infinity beyond `f64` range, /// which the callers guard against by dividing only by positive finite values. pub(crate) fn biguint_to_f64(value: &BigUint) -> f64 { @@ -267,6 +302,14 @@ pub(crate) struct Record { pub committed_usd: Option, /// The headroom valued in USD. pub fee_headroom_usd: Option, + /// Whether Fynd beat the settled trade **without** the mock — public liquidity only. + pub without_won: Option, + /// Whether Fynd beat the settled trade **with** the mock available. + pub with_won: Option, + /// USD Fynd gained over the settled trade without the mock. Negative on a loss. + pub without_improvement_usd: Option, + /// USD Fynd gained over the settled trade with the mock available. + pub with_improvement_usd: Option, } impl Record { @@ -276,11 +319,16 @@ impl Record { pair: Option, committed_usd: Option, fee_headroom_usd: Option, + ab: AbResult, ) -> Self { Self { pair, offset_bps: observed.offset_bps, won: observed.won, + without_won: ab.without_won, + with_won: ab.with_won, + without_improvement_usd: ab.without_improvement_usd, + with_improvement_usd: ab.with_improvement_usd, committed_amount_out: observed .committed_amount_out .as_ref() @@ -309,6 +357,7 @@ mod tests { fee_headroom: won.then(|| BigUint::from(headroom)), offset_bps: None, public_best_out: None, + without: None, } } diff --git a/tools/hindsight/src/report/aggregate.rs b/tools/hindsight/src/report/aggregate.rs index 54bac5e7..dcf22b32 100644 --- a/tools/hindsight/src/report/aggregate.rs +++ b/tools/hindsight/src/report/aggregate.rs @@ -66,6 +66,106 @@ pub(crate) struct PropAmm { pub by_pair: Vec, /// Per-offset groups, ascending. Empty for a run with no calibrated orders. pub groups: Vec, + /// The same orders scored with and without the mock — what the pool actually bought us. + pub uplift: Uplift, +} + +/// The controlled A/B: the same orders, at the same block states, solved with the mock available +/// and with it neutralised. +/// +/// This is the only place the report answers "did the `PropAMM` help", because it is the only +/// comparison where nothing else varies. It covers calibrated orders alone — an off-pair order has +/// no "without" pass, since the mock could never have served it. +#[derive(Debug, Default, Clone, Copy)] +pub(crate) struct Uplift { + /// Orders scored in both worlds. + pub orders: usize, + /// Of those, how many Fynd won without the mock. + pub wins_without: usize, + /// Of those, how many Fynd won with the mock. + pub wins_with: usize, + /// USD gained over the settled trades without the mock, summed over winning orders. + pub profit_without_usd: f64, + /// USD gained over the settled trades with the mock, summed over winning orders. + pub profit_with_usd: f64, +} + +impl Uplift { + /// Win rate without the mock, as a percentage. Zero when nothing was scored. + // Precision loss is irrelevant: these are order counts, far below f64's exact-integer range. + #[expect(clippy::cast_precision_loss)] + pub(crate) fn winrate_without_pct(&self) -> f64 { + if self.orders == 0 { + return 0.0; + } + self.wins_without as f64 / self.orders as f64 * 100.0 + } + + /// Win rate with the mock, as a percentage. Zero when nothing was scored. + // Precision loss is irrelevant: these are order counts, far below f64's exact-integer range. + #[expect(clippy::cast_precision_loss)] + pub(crate) fn winrate_with_pct(&self) -> f64 { + if self.orders == 0 { + return 0.0; + } + self.wins_with as f64 / self.orders as f64 * 100.0 + } + + /// Extra orders won because the mock was there. + pub(crate) fn extra_wins(&self) -> i64 { + i64::try_from(self.wins_with).unwrap_or(i64::MAX) - + i64::try_from(self.wins_without).unwrap_or(i64::MAX) + } + + /// Extra USD earned because the mock was there. + pub(crate) fn extra_profit_usd(&self) -> f64 { + self.profit_with_usd - self.profit_without_usd + } + + /// Extra profit as a percentage of what the public market alone earned. `None` when the public + /// side earned nothing, where the ratio has no meaning. + pub(crate) fn profit_uplift_pct(&self) -> Option { + (self.profit_without_usd.abs() > f64::EPSILON) + .then(|| self.extra_profit_usd() / self.profit_without_usd.abs() * 100.0) + } +} + +impl PropAmm { + /// The run's overall verdict: the worst of its groups. + /// + /// One failing group fails the run — a violation at any price is a violation, and averaging it + /// against the groups that passed would bury it. + pub(crate) fn verdict(&self) -> GroupVerdict { + if let Some(failure) = self + .groups + .iter() + .find_map(|group| match &group.verdict { + GroupVerdict::Fail(reason) => Some(reason.clone()), + GroupVerdict::Pass | GroupVerdict::NoData => None, + }) + { + return GroupVerdict::Fail(failure); + } + if self + .groups + .iter() + .any(|group| group.verdict == GroupVerdict::Pass) + { + GroupVerdict::Pass + } else { + GroupVerdict::NoData + } + } + + /// Groups that reached a conclusion, over groups that have orders — "4 of 5 checks conclusive". + pub(crate) fn conclusive(&self) -> (usize, usize) { + let decided = self + .groups + .iter() + .filter(|group| group.verdict != GroupVerdict::NoData) + .count(); + (decided, self.groups.len()) + } } /// Whether an offset group behaved as its price implies. @@ -265,9 +365,48 @@ fn propamm(records: &[Comparison]) -> Option { median_headroom_bps: median(&mut headroom_bps), by_pair: propamm_by_pair(&scoped), groups: propamm_groups(&scoped), + uplift: uplift(&scoped), }) } +/// Sums the with/without A/B over the orders that carry both sides. +/// +/// Wins and profits are counted independently: an order can be scored for its verdict but have an +/// unpriced output token, so it contributes to the win counts and not to the USD. +fn uplift(scoped: &[&Comparison]) -> Uplift { + let mut uplift = Uplift::default(); + for propamm in scoped + .iter() + .filter_map(|r| r.propamm.as_ref()) + { + let (Some(without_won), Some(with_won)) = (propamm.without_won, propamm.with_won) else { + continue; + }; + uplift.orders += 1; + uplift.wins_without += usize::from(without_won); + uplift.wins_with += usize::from(with_won); + // Only winning orders contribute profit: a loss is not negative revenue, it is a trade Fynd + // would not have served, which is what the win counts already say. + if without_won { + if let Some(usd) = propamm + .without_improvement_usd + .filter(|usd| usd.is_finite()) + { + uplift.profit_without_usd += usd; + } + } + if with_won { + if let Some(usd) = propamm + .with_improvement_usd + .filter(|usd| usd.is_finite()) + { + uplift.profit_with_usd += usd; + } + } + } + uplift +} + /// Groups the calibrated orders by offset and judges each group against its price. /// /// Orders with no offset are excluded: they were not calibrated, so no expectation applies to them. @@ -947,4 +1086,109 @@ mod tests { }; assert!(group.selected_pct().abs() < f64::EPSILON); } + + /// A calibrated record carrying the with/without A/B. + fn ab_record( + block: u64, + without_won: bool, + with_won: bool, + without_usd: f64, + with_usd: f64, + ) -> Comparison { + serde_json::from_value(serde_json::json!({ + "block": block, + "settled_tx": format!("0x{block:064x}"), + "venue": "relay", "solver": "1inch", + "token_in": "0xeth", "token_out": "0xusdc", + "top": { "verdict": "win", "net_bps": 5.0, "settled_value_usd": 1000.0 }, + "propamm": { + "pair": "ETH/USDC", "offset_bps": 5, "won": with_won && !without_won, + "without_won": without_won, "with_won": with_won, + "without_improvement_usd": without_usd, "with_improvement_usd": with_usd, + }, + })) + .unwrap() + } + + #[test] + fn test_uplift_counts_wins_in_both_worlds() { + let records = vec![ + ab_record(1, false, true, -2.0, 3.0), // the mock turned a loss into a win + ab_record(2, true, true, 4.0, 6.0), // already won; the mock made it worth more + ab_record(3, false, false, -1.0, -1.0), // the mock could not help + ]; + let uplift = propamm(&records).unwrap().uplift; + + assert_eq!(uplift.orders, 3); + assert_eq!(uplift.wins_without, 1); + assert_eq!(uplift.wins_with, 2); + assert_eq!(uplift.extra_wins(), 1); + } + + #[test] + fn test_uplift_profit_counts_winning_orders_only() { + // A loss is not negative revenue — it is a trade Fynd would not have served, which the win + // counts already express. Summing losing orders' USD would understate both columns. + let records = + vec![ab_record(1, false, true, -2.0, 3.0), ab_record(2, true, true, 4.0, 6.0)]; + let uplift = propamm(&records).unwrap().uplift; + + assert!( + (uplift.profit_without_usd - 4.0).abs() < 1e-9, + "the lost order contributes nothing" + ); + assert!((uplift.profit_with_usd - 9.0).abs() < 1e-9); + assert!((uplift.extra_profit_usd() - 5.0).abs() < 1e-9); + } + + #[test] + fn test_uplift_winrates_are_over_the_same_orders() { + let records = vec![ + ab_record(1, false, true, 0.0, 1.0), + ab_record(2, false, true, 0.0, 1.0), + ab_record(3, false, false, 0.0, 0.0), + ab_record(4, false, false, 0.0, 0.0), + ]; + let uplift = propamm(&records).unwrap().uplift; + assert!(uplift.winrate_without_pct().abs() < f64::EPSILON); + assert!((uplift.winrate_with_pct() - 50.0).abs() < 1e-9); + } + + #[test] + fn test_uplift_skips_orders_scored_in_only_one_world() { + // An off-pair order has no "without" pass; including it would inflate the denominator with + // orders the mock could never have served. + let records = vec![ + propamm_record(1, "0xdai", "0xusdc", false, 0.0, 0.0), + ab_record(2, false, true, 0.0, 2.0), + ]; + let uplift = propamm(&records).unwrap().uplift; + assert_eq!(uplift.orders, 1); + } + + #[test] + fn test_uplift_pct_is_undefined_when_the_public_side_earned_nothing() { + // Dividing by a zero baseline has no meaning; the absolute delta still does. + let records = vec![ab_record(1, false, true, 0.0, 5.0)]; + let uplift = propamm(&records).unwrap().uplift; + assert!(uplift.profit_uplift_pct().is_none()); + assert!((uplift.extra_profit_usd() - 5.0).abs() < 1e-9); + } + + #[test] + fn test_uplift_reports_a_regression_as_negative() { + // If the mock ever made things worse, the numbers must say so rather than clamp at zero. + let records = vec![ab_record(1, true, false, 5.0, 0.0)]; + let uplift = propamm(&records).unwrap().uplift; + assert_eq!(uplift.extra_wins(), -1); + assert!(uplift.extra_profit_usd() < 0.0); + } + + #[test] + fn test_empty_uplift_reports_zero_rather_than_nan() { + let uplift = Uplift::default(); + assert!(uplift.winrate_without_pct().abs() < f64::EPSILON); + assert!(uplift.winrate_with_pct().abs() < f64::EPSILON); + assert_eq!(uplift.extra_wins(), 0); + } } diff --git a/tools/hindsight/src/report/html.rs b/tools/hindsight/src/report/html.rs index edd8862c..1e378f2f 100644 --- a/tools/hindsight/src/report/html.rs +++ b/tools/hindsight/src/report/html.rs @@ -52,11 +52,13 @@ fn verdict_name(verdict: &str) -> String { /// the report says which slice of trades it covers. pub(crate) fn render(report: &Report, filter: Option<&str>) -> String { let mut html = String::from(HEAD); - html.push_str(&hero_section(&report.savings, &report.summary, filter)); - html.push_str(&verdict_section(&report.verdicts)); + // The PropAMM verdict leads when there is one: a calibrated run exists to answer that question, + // and everything below it is context. if let Some(propamm) = report.propamm.as_ref() { html.push_str(&propamm_section(propamm)); } + html.push_str(&hero_section(&report.savings, &report.summary, filter)); + html.push_str(&verdict_section(&report.verdicts)); html.push_str(&trades_section("Top savings", &report.top_wins)); html.push_str(&trades_section("Biggest losses", &report.top_losses)); html.push_str(&group_section("By solver", &report.by_solver)); @@ -272,49 +274,158 @@ fn token_section(tokens: &[Count]) -> String { section("Unsolved token tail", &table) } -/// The mock-`PropAMM` section: what the exclusive route won, and what fee it could have charged on -/// top of the flow it took. +/// The mock-`PropAMM` section: did the pool behave the way its price says it should? /// -/// Rendered only for a run driven with `monitor --propamm-pair`. The headline winrate is over every -/// solved order in the run, most of which never touch the mirrored pair — so it reads low by -/// construction, and the per-pair table below is where the real answer is. Both are shown rather -/// than only the flattering one. +/// Leads with one giant verdict, because a calibrated run has a right answer and the reader should +/// not have to derive it. Each group then gives its own number at a glance, coloured by outcome and +/// labelled in words — colour is never the only carrier. The narrative sits at the bottom for +/// anyone who needs to know what "fee headroom" means. fn propamm_section(propamm: &PropAmm) -> String { let pair = propamm .pair .as_deref() .unwrap_or("unknown pair"); + let (decided, total) = propamm.conclusive(); + let (headline, cls, sub) = match propamm.verdict() { + GroupVerdict::Pass => ( + "PASS".to_string(), + "pos big", + format!("{decided} of {total} price groups conclusive, none violated"), + ), + GroupVerdict::Fail(reason) => ("FAIL".to_string(), "neg big", reason), + GroupVerdict::NoData => ( + "NO DATA".to_string(), + "idlenum big", + "no price group reached a conclusion — the run needs more orders on this pair" + .to_string(), + ), + }; + let body = format!( + "
\ +
{headline}
\ +
mock PropAMM on {}
\ +
{}
\ +
{}{}{}{}\ +

Every order on this pair is priced against the route Fynd would otherwise \ + have quoted, so each one is a test rather than a measurement. Fee headroom is the \ + fee the signed extension could have charged and still beaten that route. Quotes from this \ + pool are not executable.

", + escape(pair), + escape(&sub), + propamm_uplift(propamm), + propamm_totals(propamm), + propamm_groups(&propamm.groups), + propamm_pair_table(&propamm.by_pair), + ); + section("Mock PropAMM (exclusive route)", &body) +} + +/// The controlled A/B: the same orders solved with the mock available and with it neutralised. +/// +/// The two `with` figures are the ones to read, and the deltas beside them are what the pool +/// bought. Green means the mock helped; a negative delta would mean it cost us, which the colour +/// has to be able to say as plainly as the win. +fn propamm_uplift(propamm: &PropAmm) -> String { + let uplift = &propamm.uplift; + if uplift.orders == 0 { + return "

No order was scored in both worlds, so there is nothing to \ + compare yet.

" + .to_string(); + } + // Takes the sign as an ordering, so a count and a float both colour the same way without either + // being cast to the other. + let delta = |sign: std::cmp::Ordering, formatted: &str| { + let cls = match sign { + std::cmp::Ordering::Greater => "pos", + std::cmp::Ordering::Less => "neg", + std::cmp::Ordering::Equal => "idlenum", + }; + format!("{}", escape(formatted)) + }; + let sign_of = |value: f64| { + value + .partial_cmp(&0.0) + .unwrap_or(std::cmp::Ordering::Equal) + }; + let extra_wins = uplift.extra_wins(); + let extra_profit = uplift.extra_profit_usd(); + let profit_pct = uplift + .profit_uplift_pct() + .map_or_else(|| "—".to_string(), |pct| format!("{pct:+.0}%")); + + let winrate_without = format!("{:.1}%", uplift.winrate_without_pct()); + let winrate_with = format!("{:.1}%", uplift.winrate_with_pct()); + let winrate_delta = + format!("{:+.1} pts", uplift.winrate_with_pct() - uplift.winrate_without_pct()); + let wins_delta = format!("{extra_wins:+}"); + let profit_delta = + format!("{}{}", if extra_profit >= 0.0 { "+" } else { "-" }, fmt_usd(extra_profit.abs())); + + format!( + "\ + \ + \ + \ + \ + \ + \ + \ + \ + \ +
without PropAMMwith PropAMMdifference
win rate{}{}{}
orders won{}{}{}
quoted output vs settled{}{}{} ({})
fee captured for LPs{}{}{}
\ +

Same {} orders, same block states — the mock's presence is the only \ + difference. The first three rows are the taker's side and barely move by design: the router \ + pins a surplus quote's output to the public reference, so the underbid is not passed on. \ + It lands in the last row instead, on {} of flow the pool took.

", + winrate_without, + winrate_with, + delta( + sign_of(uplift.winrate_with_pct() - uplift.winrate_without_pct()), + &winrate_delta, + ), + uplift.wins_without, + uplift.wins_with, + delta(extra_wins.cmp(&0), &wins_delta), + fmt_usd(uplift.profit_without_usd), + fmt_usd(uplift.profit_with_usd), + delta(sign_of(extra_profit), &profit_delta), + escape(&profit_pct), + fmt_usd(0.0), + fmt_usd(propamm.fee_headroom_usd), + delta( + sign_of(propamm.fee_headroom_usd), + &format!("+{}", fmt_usd(propamm.fee_headroom_usd)), + ), + uplift.orders, + fmt_usd(propamm.captured_flow_usd), + ) +} + +/// The run's totals under the verdict: how much flow the pool took and what fee it took on it. +/// +/// Kept small and beneath the verdict — these are the size of the result, not the result. +fn propamm_totals(propamm: &PropAmm) -> String { let stat = |value: &str, label: &str| { format!( "
{value}
\
{label}
" ) }; - let headline = format!( + format!( "
{}{}{}{}{}
", - stat(&pct(propamm.won, propamm.solved), "win rate (all solved orders)"), - stat(&fmt_count(propamm.won), "wins"), + stat(&fmt_count(propamm.won), "orders won"), + stat(&pct(propamm.won, propamm.solved), "of orders solved"), stat(&fmt_usd(propamm.captured_flow_usd), "flow captured"), - stat(&fmt_bps(propamm.median_headroom_bps), "median fee headroom bps"), - stat(&fmt_bps(propamm.avg_headroom_bps()), "fee headroom bps (flow-weighted)"), - ); - let body = format!( - "

Mirrored pool: {}, priced fee-free against the public \ - best route for each order. Fee headroom is what the signed extension could have \ - charged and still beaten that route — measured, not assumed. Quotes from this pool are not \ - executable.

{}{headline}{}", - escape(pair), - propamm_groups(&propamm.groups), - propamm_pair_table(&propamm.by_pair), - ); - section("Mock PropAMM (exclusive route)", &body) + stat(&fmt_bps(propamm.median_headroom_bps), "median fee headroom"), + stat(&fmt_bps(propamm.avg_headroom_bps()), "fee headroom, flow-weighted"), + ) } /// The offset groups, one card each, ascending by price. /// -/// Each card leads with its verdict rather than its numbers: the whole point of calibrating every -/// order is that the expected behaviour is known up front, so the reader should not have to work -/// out whether 5.2% is good. The numbers are underneath for when it fails. +/// Each card's big number is the count the group's expectation is about: for a below-market group +/// that is how many times it was wrongly selected (so zero is the good answer), and above market it +/// is how many times it won. The number is coloured by outcome and always sits beside a word. fn propamm_groups(groups: &[PropAmmGroup]) -> String { if groups.is_empty() { return "

No calibrated orders — no settled trade in this run was on \ @@ -323,10 +434,10 @@ fn propamm_groups(groups: &[PropAmmGroup]) -> String { } let mut cards = String::new(); for group in groups { - let (chip, cls) = match &group.verdict { - GroupVerdict::Pass => ("pass".to_string(), "ok"), - GroupVerdict::Fail(_) => ("FAIL".to_string(), "bad"), - GroupVerdict::NoData => ("no data".to_string(), "idle"), + let (word, cls) = match &group.verdict { + GroupVerdict::Pass => ("pass", "pos"), + GroupVerdict::Fail(_) => ("FAIL", "neg"), + GroupVerdict::NoData => ("no data", "idlenum"), }; let detail = match &group.verdict { GroupVerdict::Fail(reason) => { @@ -337,28 +448,36 @@ fn propamm_groups(groups: &[PropAmmGroup]) -> String { let _ = write!( cards, "

\ -
{}\ - {chip}
\ -
{}
\ - \ - \ - \ - \ - \ -
orders{}
selected{} ({})
fee taken (median){}
fee taken (max){}
{detail}\ +
{}
\ +
{word}
\ +
{}
\ +
{} ({})
\ +
fee taken {} median, {} max, \ + of at most {}
\ +
{}
{detail}\
", escape(&fmt_offset(group.offset_bps)), - escape(group.expectation()), - group.orders, group.selected, - share_pct_of(group.selected_pct()), - fmt_bps(group.median_fee_bps), - fmt_bps(group.max_fee_bps), + escape(&format!("selected of {} orders", group.orders)), + escape(&share_pct_of(group.selected_pct())), + escape(&fmt_bps(group.median_fee_bps)), + escape(&fmt_bps(group.max_fee_bps)), + escape(&fmt_offset_ceiling(group.offset_bps)), + escape(group.expectation()), ); } format!("
{cards}
") } +/// The fee ceiling a group's price implies, for the "of at most" caption. +fn fmt_offset_ceiling(offset_bps: i32) -> String { + if offset_bps <= 0 { + "0.0 bps".to_string() + } else { + format!("{offset_bps}.0 bps") + } +} + /// An offset as a signed bps label, so a column header reads as a price and not a bare number. fn fmt_offset(offset_bps: i32) -> String { match offset_bps.cmp(&0) { @@ -541,19 +660,35 @@ section { background: #211a30; border: 1px solid #362b4a; border-radius: 8px; .nodata { color: #9a8bbf; margin: 0; } /* A section's framing sentence: what the numbers below mean, before they are read. */ .note { color: #b9adcf; margin: 0 0 .5rem; max-width: 68ch; line-height: 1.55; } +/* The run's verdict, sized like the report's headline savings figure. */ +.propammhead { display: flex; flex-wrap: wrap; align-items: baseline; gap: 2rem; } +.propammsub { color: #b9adcf; font-size: .9rem; max-width: 52ch; line-height: 1.5; } +.idlenum { color: #9a8bbf; } /* One card per offset group. They wrap rather than scroll, so a wider ladder stays readable. */ -.groups { display: flex; flex-wrap: wrap; gap: 1rem; margin: 1.25rem 0; } +.groups { display: flex; flex-wrap: wrap; gap: 1rem; margin: 1.5rem 0; } .group { flex: 1 1 15rem; border: 1px solid #2f2540; border-radius: 6px; padding: .9rem 1rem; } .grouphead { display: flex; align-items: center; justify-content: space-between; gap: .75rem; } -.groupoff { font-weight: 600; } -.groupexp { color: #9a8bbf; font-size: .75rem; margin-top: .2rem; min-height: 2.2em; } -.groupnums { margin-top: .6rem; width: 100%; } -.groupnums td { padding: .15rem 0; border: 0; } +.groupoff { font-weight: 600; font-size: .9rem; } +/* The number the group's expectation is about, big enough to read across a room. */ +.groupbig { font-size: 2.6rem; font-weight: 700; line-height: 1.05; margin-top: .5rem; } +.groupcap { color: #9a8bbf; font-size: .78rem; } +.groupfee { color: #b9adcf; font-size: .82rem; margin-top: .6rem; } +.groupexp { color: #6f6390; font-size: .72rem; margin-top: .5rem; } /* The verdict never rests on colour alone: each chip is also labelled pass / FAIL / no data. */ -.chip.ok { background: #1b3a1f; color: #a5d6a7; } -.chip.bad { background: #4a1c1c; color: #ef9a9a; } -.chip.idle { background: #2f2540; color: #9a8bbf; } -.groupfail { color: #ef9a9a; font-size: .78rem; margin: .5rem 0 0; line-height: 1.45; } +.chip.poschip { background: #1b3a1f; color: #a5d6a7; } +.chip.negchip { background: #4a1c1c; color: #ef9a9a; } +.chip.idlenumchip { background: #2f2540; color: #9a8bbf; } +.groupfail { color: #ef9a9a; font-size: .78rem; margin: .6rem 0 0; line-height: 1.45; } +/* Each sub-test carries its own verdict word, sized so the three read at a glance together. */ +.groupverdict { font-size: 1.5rem; font-weight: 700; letter-spacing: .04em; margin-top: .45rem; } +/* The with/without table: the "with" column is the answer, so it carries the weight. */ +.ab { margin: 1.5rem 0 .25rem; } +.ab th { color: #9a8bbf; font-weight: 500; } +.ab td.strong { font-weight: 700; font-size: 1.05rem; } +.delta { font-weight: 600; } +.abnote { color: #6f6390; font-size: .76rem; margin: 0 0 .5rem; max-width: 74ch; line-height: 1.5; } +/* Where the underbid actually lands, set apart from the taker-side rows above it. */ +.ab tr.abfee td { border-top: 1px solid #2f2540; padding-top: .55rem; } table { border-collapse: collapse; width: 100%; } th, td { text-align: left; padding: .4rem .6rem; border-bottom: 1px solid #362b4a; } th { color: #9a8bbf; font-weight: 600; font-size: .8rem; text-transform: uppercase; } @@ -741,7 +876,7 @@ mod tests { let html = render(&propamm_report(true), None); assert!(html.contains("Mock PropAMM (exclusive route)")); assert!(html.contains("WETH/USDC"), "the section must name the pool it stood in for"); - assert!(html.contains("median fee headroom bps")); + assert!(html.contains("median fee headroom")); // The order pair is rendered short, with the full addresses in the hover title. assert!(html.contains("0xc02aaa39…756cc2")); assert!(html.contains("0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2")); @@ -771,6 +906,9 @@ mod tests { "fee_headroom_bps": won.then_some(fee), "committed_usd": won.then_some(1_000.0), "fee_headroom_usd": won.then_some(fee / 10.0), + "without_won": false, "with_won": won, + "without_improvement_usd": -1.0, + "with_improvement_usd": if won { 4.0 } else { -1.0 }, }, }) }; @@ -785,6 +923,26 @@ mod tests { build(&records) } + #[test] + fn test_propamm_section_leads_the_document() { + // A calibrated run exists to answer one question, so its verdict comes before the savings + // headline rather than three sections down. + let html = render(&group_report(5.0), None); + let propamm = html + .find("Mock PropAMM") + .expect("section present"); + let savings = html + .find("Fynd savings") + .expect("hero present"); + assert!(propamm < savings, "the PropAMM verdict must precede the savings headline"); + } + + #[test] + fn test_overall_verdict_is_a_single_word() { + assert!(render(&group_report(5.0), None).contains(">PASS<")); + assert!(render(&group_report(20.0), None).contains(">FAIL<")); + } + #[test] fn test_group_cards_lead_with_the_verdict_and_name_each_price() { let html = render(&group_report(5.0), None); @@ -814,4 +972,32 @@ mod tests { let html = render(&propamm_report(true), None); assert!(html.contains("No calibrated orders")); } + + #[test] + fn test_uplift_table_shows_both_worlds_and_the_difference() { + let html = render(&group_report(5.0), None); + assert!(html.contains("without PropAMM")); + assert!(html.contains("with PropAMM")); + assert!(html.contains("win rate")); + assert!(html.contains("quoted output vs settled")); + assert!(html.contains("fee captured for LPs")); + // The difference must be signed, so a regression reads as one. + assert!(html.contains("delta"), "each row carries a signed difference"); + } + + #[test] + fn test_each_group_carries_its_own_verdict_word() { + // Three sub-tests, three verdicts — the overall PASS must not be the only one visible. + let html = render(&group_report(5.0), None); + let verdicts = html + .matches("class=\"groupverdict") + .count(); + assert_eq!(verdicts, 3, "one verdict per offset group"); + } + + #[test] + fn test_uplift_table_says_so_when_nothing_was_scored_in_both_worlds() { + let html = render(&propamm_report(true), None); + assert!(html.contains("nothing to"), "an empty A/B must explain itself"); + } } diff --git a/tools/hindsight/src/report/record.rs b/tools/hindsight/src/report/record.rs index 2dc80e9e..2727a2b2 100644 --- a/tools/hindsight/src/report/record.rs +++ b/tools/hindsight/src/report/record.rs @@ -50,6 +50,18 @@ pub(crate) struct PropAmm { /// The headroom valued in USD. #[serde(default)] pub fee_headroom_usd: Option, + /// Whether Fynd beat the settled trade **without** the mock — public liquidity only. + #[serde(default)] + pub without_won: Option, + /// Whether Fynd beat the settled trade **with** the mock available. + #[serde(default)] + pub with_won: Option, + /// USD Fynd gained over the settled trade without the mock. Negative on a loss. + #[serde(default)] + pub without_improvement_usd: Option, + /// USD Fynd gained over the settled trade with the mock available. + #[serde(default)] + pub with_improvement_usd: Option, } /// Fynd's result at one block state. @@ -168,9 +180,23 @@ mod tests { fee_headroom: Some(400_000u64.into()), offset_bps: Some(5), public_best_out: Some(1_000_000_000u64.into()), + without: Some(crate::propamm::report::PublicOnly { + amount_out: 1_000_000_000u64.into(), + amount_out_net_gas: 999_000_000u64.into(), + }), }; - let record = - Record::new(&observed, Some("WETH/USDC".to_string()), Some(1_000.0), Some(0.4)); + let record = Record::new( + &observed, + Some("WETH/USDC".to_string()), + Some(1_000.0), + Some(0.4), + crate::propamm::report::AbResult { + without_won: Some(false), + with_won: Some(true), + without_improvement_usd: Some(-1.0), + with_improvement_usd: Some(3.0), + }, + ); let line = serde_json::to_string(&serde_json::json!({ "block": 1, diff --git a/tools/hindsight/src/resolve/monitor.rs b/tools/hindsight/src/resolve/monitor.rs index b97a6fda..11e9c023 100644 --- a/tools/hindsight/src/resolve/monitor.rs +++ b/tools/hindsight/src/resolve/monitor.rs @@ -148,6 +148,14 @@ pub(crate) struct MonitorArgs { )] pub propamm_offsets_bps: Vec, + /// Re-solve only the settled trades whose own pair is the mirrored one — the only orders the + /// harness can calibrate. Off-pair trades are the large majority, so skipping them fills the + /// offset groups far faster per block. The trade-off: that run's comparisons cover the + /// mirrored pair alone, so its savings and coverage numbers are not a picture of Fynd + /// overall + #[arg(long, env = "PROPAMM_ONLY_PAIR", default_value_t = false)] + pub propamm_only_pair: bool, + /// Trade size, in whole units of the pair's first token, used each block to pick which real /// pool to mirror. Set it near the sizes being re-solved so the mirror tracks the pool that /// actually prices those trades best @@ -375,7 +383,7 @@ impl StepAdapter<'_> { .calibrate( &market, &core_token_in, - &amount_in_biguint(amount_in), + &u256_to_biguint(amount_in), &public_best_out, offset_bps, ) @@ -406,7 +414,11 @@ impl StepAdapter<'_> { let (outcome, observed) = self .quote_order(token_in, token_out, amount_in) .await; - let observed = observed.map(|o| o.with_calibration(&calibration)); + let without = propamm::report::PublicOnly { + amount_out: public_best_out.clone(), + amount_out_net_gas: u256_to_biguint(public.amount_out_net_gas), + }; + let observed = observed.map(|o| o.with_calibration(&calibration, without)); (outcome, observed) } } @@ -858,8 +870,9 @@ fn report_propamm_block( block: u64, prices: &Prices, observations: &[propamm::report::Observation], - trade_count: usize, + ranges: &[RangeComparison], ) -> Vec> { + let trade_count = ranges.len(); let mut headroom_usd = 0.0; let mut captured_flow_usd = 0.0; let mut records: Vec> = Vec::with_capacity(observations.len()); @@ -882,11 +895,20 @@ fn report_propamm_block( let committed = value_usd(observed.committed_amount_out.as_ref()); headroom_usd += headroom.unwrap_or(0.0); captured_flow_usd += committed.unwrap_or(0.0); + // The A/B is only defined for calibrated orders, and only against the range this + // observation pairs with — the first `trade_count` observations are the + // top-of-block ones. + let ab = ranges + .get(records.len()) + .map_or_else(propamm::report::AbResult::default, |range| { + score_both_worlds(observed, range, prices) + }); records.push(Some(propamm::report::Record::new( observed, pair_label.clone(), committed, headroom, + ab, ))); } @@ -924,14 +946,85 @@ fn report_propamm_block( vec![None; trade_count] } +/// Narrows a block's trades to the mirrored pair when `--propamm-only-pair` is set. +/// +/// The count is logged rather than dropped quietly: a run that silently skipped most of a block +/// would read as a block with little flow. +fn scope_trades( + trades: Vec, + adapter: &StepAdapter<'_>, + only_pair: bool, + block: u64, +) -> Vec { + let Some(harness) = adapter.propamm.filter(|_| only_pair) else { + return trades; + }; + let total = trades.len(); + let kept = retain_mirrored_pair(trades, harness.config()); + debug!( + block, + kept = kept.len(), + skipped = total - kept.len(), + "--propamm-only-pair: re-solving the mirrored pair's trades only" + ); + kept +} + +/// Scores one order in both worlds against what actually settled. +/// +/// "Without" is the pass the calibration already ran with the mock neutralised; "with" is the +/// range's own top-of-block result. Both are compared to the settled trade the same way the +/// headline verdict is — output net of gas — so the pair is directly comparable and the difference +/// is the mock's entire contribution. +fn score_both_worlds( + observed: &propamm::report::Observation, + range: &RangeComparison, + prices: &Prices, +) -> propamm::report::AbResult { + let Some(without) = observed.without.as_ref() else { + return propamm::report::AbResult::default(); + }; + let settled_net_gas = u256_to_biguint(range.settled_amount_out_net_gas); + let with = match &range.top.outcome { + Outcome::Solved(solved) => Some(solved), + Outcome::Partial(_) | Outcome::Unsolvable(_) => None, + }; + let improvement = |amount_out: &BigUint| { + biguint_to_u256_opt(amount_out) + .and_then(|out| prices.savings_usd(range.token_out, out, range.settled_amount_out)) + }; + propamm::report::AbResult { + without_won: Some(without.amount_out_net_gas > settled_net_gas), + with_won: with.map(|s| u256_to_biguint(s.amount_out_net_gas) > settled_net_gas), + without_improvement_usd: improvement(&without.amount_out), + with_improvement_usd: with.and_then(|s| improvement(&u256_to_biguint(s.amount_out))), + } +} + +/// Keeps only the trades whose own token pair is the mirrored one, in either direction. +fn retain_mirrored_pair( + trades: Vec, + config: &propamm::MirrorConfig, +) -> Vec { + trades + .into_iter() + .filter(|trade| { + config.serves( + &CoreAddress::from(trade.token_in.into_array()), + &CoreAddress::from(trade.token_out.into_array()), + ) + }) + .collect() +} + /// The solver's market-data handle, which the injector writes the mock's state through. fn harness_market(solver: &Solver) -> fynd_core::feed::market_data::MarketData { solver.market_data() } -/// Converts a `U256` order size to `BigUint`, which is what the mock's calibration math uses. -fn amount_in_biguint(amount_in: U256) -> BigUint { - BigUint::from_bytes_be(&amount_in.to_be_bytes::<32>()) +/// Converts a `U256` amount to `BigUint`, which is what the mock's calibration math uses. +fn u256_to_biguint(amount: U256) -> BigUint { + BigUint::from_bytes_be(&amount.to_be_bytes::<32>()) } /// Converts a `BigUint` to `U256`, or `None` when it does not fit — the amount is then left @@ -976,7 +1069,7 @@ fn emit_block_results( let propamm_records = match adapter.propamm { Some(harness) => { let observations = harness.stats.drain(); - report_propamm_block(harness, block, prices_top, &observations, ranges.len()) + report_propamm_block(harness, block, prices_top, &observations, ranges) } None => vec![None; ranges.len()], }; @@ -1061,6 +1154,8 @@ async fn run_session( } }; + let trades = scope_trades(trades, adapter, cfg.propamm_only_pair, target); + let start = Instant::now(); // Snapshot token prices at top-of-block (N-1) for the headline metric and the top-of-block // USD valuation. @@ -1243,9 +1338,54 @@ mod tests { comparisons_dir: None, propamm_pair: None, propamm_offsets_bps: vec![-5, 0, 5], + propamm_only_pair: false, propamm_probe_units: 1.0, }) .await .expect("monitor should process one block without error"); } + + #[test] + fn test_retain_mirrored_pair_keeps_both_directions_only() { + let eth = Address::ZERO; + let usdc = Address::repeat_byte(0xaa); + let dai = Address::repeat_byte(0xbb); + let config = propamm::MirrorConfig { + token_a: CoreAddress::from(eth.into_array()), + token_b: CoreAddress::from(usdc.into_array()), + offsets_bps: vec![-5, 0, 5], + probe_units: 1.0, + chain: Chain::Ethereum, + }; + let trade = |token_in: Address, token_out: Address| DecodedTrade { + tx_hash: alloy::primitives::TxHash::default(), + block_number: 1, + tx_index: 0, + venue: "relay".into(), + solver: "tycho".into(), + solver_source: crate::decoder::AttributionSource::TraceMatch, + decoder: "sender-netting", + sender: Address::ZERO, + token_in, + token_out, + amount_in: U256::from(1u64), + amount_out: U256::from(1u64), + venue_fee_in: None, + venue_fee_out: None, + settled_gas: None, + quote: None, + sandwich: None, + }; + + let kept = retain_mirrored_pair( + vec![ + trade(eth, usdc), // mirrored, forward + trade(usdc, eth), // mirrored, reverse + trade(dai, usdc), // off-pair + trade(eth, dai), // shares one token only + ], + &config, + ); + assert_eq!(kept.len(), 2, "only the mirrored pair survives, in either direction"); + } } From b1dd8e7cae80a06110884e2b669eaa0926f12f2d Mon Sep 17 00:00:00 2001 From: TAMARA LIPOWSKI Date: Tue, 28 Jul 2026 21:10:49 -0400 Subject: [PATCH 10/12] feat(hindsight): split the report headline into with and without PropAMM The hero now shows Fynd's savings, win rate and median savings bps twice, once per world, with the deltas attached to the 'with' column and the LP capture beside them. Both worlds are computed on the same basis the headline already used: a win is gross output above settled, and the bps figure is net of each side's own gas. Drops the per-pair table and the prose in favour of the numbers, and gives each offset group its own verdict alongside the overall one. --- tools/hindsight/src/propamm/report.rs | 8 + tools/hindsight/src/report/aggregate.rs | 169 ++------ tools/hindsight/src/report/html.rs | 487 +++++++++++------------- tools/hindsight/src/report/record.rs | 8 + tools/hindsight/src/resolve/monitor.rs | 15 +- 5 files changed, 265 insertions(+), 422 deletions(-) diff --git a/tools/hindsight/src/propamm/report.rs b/tools/hindsight/src/propamm/report.rs index 972a96cc..2dafe579 100644 --- a/tools/hindsight/src/propamm/report.rs +++ b/tools/hindsight/src/propamm/report.rs @@ -267,6 +267,8 @@ pub(crate) struct AbResult { pub with_won: Option, pub without_improvement_usd: Option, pub with_improvement_usd: Option, + pub without_net_bps: Option, + pub with_net_bps: Option, } /// Converts a `BigUint` to `f64` for ratio reporting. Saturates to infinity beyond `f64` range, @@ -310,6 +312,10 @@ pub(crate) struct Record { pub without_improvement_usd: Option, /// USD Fynd gained over the settled trade with the mock available. pub with_improvement_usd: Option, + /// Net-of-gas bps over the settled trade without the mock. + pub without_net_bps: Option, + /// Net-of-gas bps over the settled trade with the mock available. + pub with_net_bps: Option, } impl Record { @@ -329,6 +335,8 @@ impl Record { with_won: ab.with_won, without_improvement_usd: ab.without_improvement_usd, with_improvement_usd: ab.with_improvement_usd, + without_net_bps: ab.without_net_bps, + with_net_bps: ab.with_net_bps, committed_amount_out: observed .committed_amount_out .as_ref() diff --git a/tools/hindsight/src/report/aggregate.rs b/tools/hindsight/src/report/aggregate.rs index dcf22b32..46532b44 100644 --- a/tools/hindsight/src/report/aggregate.rs +++ b/tools/hindsight/src/report/aggregate.rs @@ -7,9 +7,6 @@ use std::collections::HashMap; use crate::report::record::Comparison; -/// Number of order pairs listed in the mock-`PropAMM` breakdown table. -const TOP_PROPAMM_PAIRS: usize = 10; - /// Slack allowed when checking a group's fee against its offset, in basis points. /// /// The calibration divides integer amounts and the source pool rounds its own output, so a fee @@ -42,28 +39,15 @@ pub(crate) struct Report { pub propamm: Option, } -/// The mock-`PropAMM` view: how often the exclusive route beat the public one, and what fee it -/// could have charged on the flow it took. -/// -/// The run-wide winrate is over every solved order, most of which never touch the mirrored pair — -/// so `by_pair` is where the answer actually lives. Each row is one order direction, and the mock -/// can serve an order whose own pair differs from the mirrored one (a `DAI→USDC` order routed -/// `DAI→WETH→USDC` uses a `WETH/USDC` leg), which is exactly what the breakdown shows. +/// The mock-`PropAMM` view: what the pool captured, whether each price group behaved, and the +/// with/without comparison. pub(crate) struct PropAmm { /// The mirrored pair as token symbols, e.g. `WETH/USDC`. pub pair: Option, - /// Orders with a scored quote — the winrate's denominator. - pub solved: usize, - /// Of those, how many routed through the mock pool. - pub won: usize, /// Committed output on wins, valued in USD — the flow the pool captured. pub captured_flow_usd: f64, /// Fee headroom on wins, valued in USD. pub fee_headroom_usd: f64, - /// Median fee headroom in bps over wins — the typical fee the pool could charge. - pub median_headroom_bps: Option, - /// Per-order-pair breakdown, most wins first. - pub by_pair: Vec, /// Per-offset groups, ascending. Empty for a run with no calibrated orders. pub groups: Vec, /// The same orders scored with and without the mock — what the pool actually bought us. @@ -88,6 +72,10 @@ pub(crate) struct Uplift { pub profit_without_usd: f64, /// USD gained over the settled trades with the mock, summed over winning orders. pub profit_with_usd: f64, + /// Median net-of-gas bps over winning orders without the mock. + pub median_bps_without: Option, + /// Median net-of-gas bps over winning orders with the mock. + pub median_bps_with: Option, } impl Uplift { @@ -121,13 +109,6 @@ impl Uplift { pub(crate) fn extra_profit_usd(&self) -> f64 { self.profit_with_usd - self.profit_without_usd } - - /// Extra profit as a percentage of what the public market alone earned. `None` when the public - /// side earned nothing, where the ratio has no meaning. - pub(crate) fn profit_uplift_pct(&self) -> Option { - (self.profit_without_usd.abs() > f64::EPSILON) - .then(|| self.extra_profit_usd() / self.profit_without_usd.abs() * 100.0) - } } impl PropAmm { @@ -156,16 +137,6 @@ impl PropAmm { GroupVerdict::NoData } } - - /// Groups that reached a conclusion, over groups that have orders — "4 of 5 checks conclusive". - pub(crate) fn conclusive(&self) -> (usize, usize) { - let decided = self - .groups - .iter() - .filter(|group| group.verdict != GroupVerdict::NoData) - .count(); - (decided, self.groups.len()) - } } /// Whether an offset group behaved as its price implies. @@ -236,24 +207,6 @@ impl PropAmm { } } -/// One order direction in the mock-`PropAMM` breakdown. -pub(crate) struct PropAmmPair { - /// `token_in` address. - pub token_in: String, - /// `token_out` address. - pub token_out: String, - /// Solved orders in this direction. - pub solved: usize, - /// Of those, how many the mock pool won. - pub won: usize, - /// Committed output on those wins, valued in USD. - pub captured_flow_usd: f64, - /// Fee headroom on those wins, valued in USD. - pub fee_headroom_usd: f64, - /// Median fee headroom in bps over those wins. - pub median_headroom_bps: Option, -} - pub(crate) struct Summary { pub total: usize, pub distinct_blocks: usize, @@ -344,26 +297,14 @@ fn propamm(records: &[Comparison]) -> Option { .is_some_and(|p| p.won) }) .collect(); - let mut headroom_bps: Vec = wins - .iter() - .filter_map(|r| { - r.propamm - .as_ref() - .and_then(|p| p.fee_headroom_bps) - }) - .collect(); Some(PropAmm { pair: scoped.iter().find_map(|r| { r.propamm .as_ref() .and_then(|p| p.pair.clone()) }), - solved: scoped.len(), - won: wins.len(), captured_flow_usd: sum_propamm(&wins, |p| p.committed_usd), fee_headroom_usd: sum_propamm(&wins, |p| p.fee_headroom_usd), - median_headroom_bps: median(&mut headroom_bps), - by_pair: propamm_by_pair(&scoped), groups: propamm_groups(&scoped), uplift: uplift(&scoped), }) @@ -375,6 +316,8 @@ fn propamm(records: &[Comparison]) -> Option { /// unpriced output token, so it contributes to the win counts and not to the USD. fn uplift(scoped: &[&Comparison]) -> Uplift { let mut uplift = Uplift::default(); + let mut bps_without: Vec = Vec::new(); + let mut bps_with: Vec = Vec::new(); for propamm in scoped .iter() .filter_map(|r| r.propamm.as_ref()) @@ -403,7 +346,17 @@ fn uplift(scoped: &[&Comparison]) -> Uplift { uplift.profit_with_usd += usd; } } + // The bps headline is over wins only, matching the report's own median: how much better + // Fynd was when it won, not diluted by the trades it lost. + if without_won { + bps_without.extend(propamm.without_net_bps); + } + if with_won { + bps_with.extend(propamm.with_net_bps); + } } + uplift.median_bps_without = median(&mut bps_without); + uplift.median_bps_with = median(&mut bps_with); uplift } @@ -515,58 +468,6 @@ fn sum_propamm( .sum() } -/// Groups the mock-`PropAMM` outcomes by order direction, ranked by wins then by solved count, so -/// the pairs the exclusive route actually served come first. -fn propamm_by_pair(scoped: &[&Comparison]) -> Vec { - let mut grouped: HashMap<(&str, &str), Vec<&Comparison>> = HashMap::new(); - for record in scoped { - grouped - .entry((record.token_in.as_str(), record.token_out.as_str())) - .or_default() - .push(record); - } - - let mut rows: Vec = grouped - .into_iter() - .map(|((token_in, token_out), records)| { - let wins: Vec<&Comparison> = records - .iter() - .copied() - .filter(|r| { - r.propamm - .as_ref() - .is_some_and(|p| p.won) - }) - .collect(); - let mut headroom_bps: Vec = wins - .iter() - .filter_map(|r| { - r.propamm - .as_ref() - .and_then(|p| p.fee_headroom_bps) - }) - .collect(); - PropAmmPair { - token_in: token_in.to_string(), - token_out: token_out.to_string(), - solved: records.len(), - won: wins.len(), - captured_flow_usd: sum_propamm(&wins, |p| p.committed_usd), - fee_headroom_usd: sum_propamm(&wins, |p| p.fee_headroom_usd), - median_headroom_bps: median(&mut headroom_bps), - } - }) - .collect(); - rows.sort_by(|a, b| { - b.won - .cmp(&a.won) - .then_with(|| b.solved.cmp(&a.solved)) - .then_with(|| a.token_in.cmp(&b.token_in)) - }); - rows.truncate(TOP_PROPAMM_PAIRS); - rows -} - fn summary(records: &[Comparison]) -> Summary { let mut blocks: Vec = records .iter() @@ -904,9 +805,9 @@ mod tests { propamm_record(3, "0xweth", "0xusdc", false, 0.0, 0.0), ]; let propamm = propamm(&records).expect("some records carry an outcome"); - assert_eq!(propamm.solved, 2); - assert_eq!(propamm.won, 1); assert_eq!(propamm.pair.as_deref(), Some("WETH/USDC")); + // Both records carry an outcome, so both are in scope; only one is a win. + assert!((propamm.captured_flow_usd - 1_000.0).abs() < 1e-6); } #[test] @@ -921,7 +822,6 @@ mod tests { assert!((propamm.captured_flow_usd - 4_000.0).abs() < 1e-6); // 1000 @ 4 bps = 0.40, 3000 @ 8 bps = 2.40. assert!((propamm.fee_headroom_usd - 2.8).abs() < 1e-6); - assert!((propamm.median_headroom_bps.unwrap() - 6.0).abs() < 1e-6); // Flow-weighted, not the mean of the two bps values: 2.8 / 4000 = 7 bps. assert!((propamm.avg_headroom_bps().unwrap() - 7.0).abs() < 1e-6); } @@ -934,28 +834,6 @@ mod tests { assert!(propamm.avg_headroom_bps().is_none()); } - #[test] - fn test_propamm_by_pair_ranks_the_served_pairs_first() { - // The mirrored pair is WETH/USDC, but the mock also serves a DAI→USDC order routed through - // WETH. The pair with wins must outrank an equally-sized pair without any. - let records = vec![ - propamm_record(1, "0xdai", "0xusdc", false, 0.0, 0.0), - propamm_record(2, "0xdai", "0xusdc", false, 0.0, 0.0), - propamm_record(3, "0xweth", "0xusdc", true, 4.0, 1_000.0), - propamm_record(4, "0xweth", "0xusdc", false, 0.0, 0.0), - ]; - let propamm = propamm(&records).expect("outcomes present"); - let rows = &propamm.by_pair; - - assert_eq!(rows.len(), 2); - assert_eq!(rows[0].token_in, "0xweth"); - assert_eq!(rows[0].solved, 2); - assert_eq!(rows[0].won, 1); - assert_eq!(rows[1].token_in, "0xdai"); - assert_eq!(rows[1].won, 0); - assert!(rows[1].median_headroom_bps.is_none()); - } - /// A calibrated record: priced `offset_bps` off the public best route, selected or not. fn calibrated(block: u64, offset_bps: i32, won: bool, fee_bps: f64) -> Comparison { serde_json::from_value(serde_json::json!({ @@ -1070,8 +948,6 @@ mod tests { let propamm = propamm(&records).expect("outcomes present"); assert_eq!(propamm.groups.len(), 1); assert_eq!(propamm.groups[0].orders, 1); - // Both still count toward the run's solved total. - assert_eq!(propamm.solved, 2); } #[test] @@ -1167,11 +1043,10 @@ mod tests { } #[test] - fn test_uplift_pct_is_undefined_when_the_public_side_earned_nothing() { - // Dividing by a zero baseline has no meaning; the absolute delta still does. + fn test_uplift_reports_an_absolute_gain_over_a_zero_baseline() { + // The public side earning nothing is not a divide-by-zero problem; the delta still stands. let records = vec![ab_record(1, false, true, 0.0, 5.0)]; let uplift = propamm(&records).unwrap().uplift; - assert!(uplift.profit_uplift_pct().is_none()); assert!((uplift.extra_profit_usd() - 5.0).abs() < 1e-9); } diff --git a/tools/hindsight/src/report/html.rs b/tools/hindsight/src/report/html.rs index 1e378f2f..9b1f3efe 100644 --- a/tools/hindsight/src/report/html.rs +++ b/tools/hindsight/src/report/html.rs @@ -9,8 +9,8 @@ use std::fmt::Write as _; use crate::report::aggregate::{ - Count, GroupStats, GroupVerdict, PropAmm, PropAmmGroup, PropAmmPair, Report, Savings, Summary, - TradeRow, VerdictStat, + Count, GroupStats, GroupVerdict, PropAmm, PropAmmGroup, Report, Savings, Summary, TradeRow, + VerdictStat, }; /// Shortest share of a stacked column that gets an inline `12.3%` label. Below it the segment is @@ -52,12 +52,12 @@ fn verdict_name(verdict: &str) -> String { /// the report says which slice of trades it covers. pub(crate) fn render(report: &Report, filter: Option<&str>) -> String { let mut html = String::from(HEAD); - // The PropAMM verdict leads when there is one: a calibrated run exists to answer that question, - // and everything below it is context. - if let Some(propamm) = report.propamm.as_ref() { - html.push_str(&propamm_section(propamm)); + // A calibrated run's headline is the with/without split, so it replaces the single-world hero + // rather than sitting beside a number that silently mixes the two. + match report.propamm.as_ref() { + Some(propamm) => html.push_str(&propamm_hero(propamm, &report.summary, filter)), + None => html.push_str(&hero_section(&report.savings, &report.summary, filter)), } - html.push_str(&hero_section(&report.savings, &report.summary, filter)); html.push_str(&verdict_section(&report.verdicts)); html.push_str(&trades_section("Top savings", &report.top_wins)); html.push_str(&trades_section("Biggest losses", &report.top_losses)); @@ -274,153 +274,135 @@ fn token_section(tokens: &[Count]) -> String { section("Unsolved token tail", &table) } -/// The mock-`PropAMM` section: did the pool behave the way its price says it should? +/// The headline for a calibrated run: the same orders scored with and without the mock, side by +/// side, plus one verdict per price group. /// -/// Leads with one giant verdict, because a calibrated run has a right answer and the reader should -/// not have to derive it. Each group then gives its own number at a glance, coloured by outcome and -/// labelled in words — colour is never the only carrier. The narrative sits at the bottom for -/// anyone who needs to know what "fee headroom" means. -fn propamm_section(propamm: &PropAmm) -> String { +/// Two columns of the same three figures the single-world hero shows, so the comparison is read by +/// scanning across rather than by reading a caption. Everything is a number: the deltas carry the +/// sign, colour carries the direction, and each figure is labelled. +fn propamm_hero(propamm: &PropAmm, summary: &Summary, filter: Option<&str>) -> String { + let uplift = &propamm.uplift; + let scope = filter.map_or_else( + || "all venues".to_string(), + |venue| format!("venue: {}", escape(venue)), + ); let pair = propamm .pair .as_deref() .unwrap_or("unknown pair"); - let (decided, total) = propamm.conclusive(); - let (headline, cls, sub) = match propamm.verdict() { - GroupVerdict::Pass => ( - "PASS".to_string(), - "pos big", - format!("{decided} of {total} price groups conclusive, none violated"), - ), - GroupVerdict::Fail(reason) => ("FAIL".to_string(), "neg big", reason), - GroupVerdict::NoData => ( - "NO DATA".to_string(), - "idlenum big", - "no price group reached a conclusion — the run needs more orders on this pair" - .to_string(), - ), + let (verdict, verdict_cls) = match propamm.verdict() { + GroupVerdict::Pass => ("PASS", "pos"), + GroupVerdict::Fail(_) => ("FAIL", "neg"), + GroupVerdict::NoData => ("NO DATA", "idlenum"), }; - let body = format!( - "
\ -
{headline}
\ -
mock PropAMM on {}
\ -
{}
\ -
{}{}{}{}\ -

Every order on this pair is priced against the route Fynd would otherwise \ - have quoted, so each one is a test rather than a measurement. Fee headroom is the \ - fee the signed extension could have charged and still beaten that route. Quotes from this \ - pool are not executable.

", - escape(pair), - escape(&sub), - propamm_uplift(propamm), - propamm_totals(propamm), - propamm_groups(&propamm.groups), - propamm_pair_table(&propamm.by_pair), - ); - section("Mock PropAMM (exclusive route)", &body) -} - -/// The controlled A/B: the same orders solved with the mock available and with it neutralised. -/// -/// The two `with` figures are the ones to read, and the deltas beside them are what the pool -/// bought. Green means the mock helped; a negative delta would mean it cost us, which the colour -/// has to be able to say as plainly as the win. -fn propamm_uplift(propamm: &PropAmm) -> String { - let uplift = &propamm.uplift; - if uplift.orders == 0 { - return "

No order was scored in both worlds, so there is nothing to \ - compare yet.

" - .to_string(); - } - // Takes the sign as an ordering, so a count and a float both colour the same way without either - // being cast to the other. - let delta = |sign: std::cmp::Ordering, formatted: &str| { - let cls = match sign { - std::cmp::Ordering::Greater => "pos", - std::cmp::Ordering::Less => "neg", - std::cmp::Ordering::Equal => "idlenum", - }; - format!("{}", escape(formatted)) - }; - let sign_of = |value: f64| { - value - .partial_cmp(&0.0) - .unwrap_or(std::cmp::Ordering::Equal) + let mini = |value: &str, label: &str| { + format!( + "
{value}
\ +
{label}
" + ) }; - let extra_wins = uplift.extra_wins(); - let extra_profit = uplift.extra_profit_usd(); - let profit_pct = uplift - .profit_uplift_pct() - .map_or_else(|| "—".to_string(), |pct| format!("{pct:+.0}%")); - - let winrate_without = format!("{:.1}%", uplift.winrate_without_pct()); - let winrate_with = format!("{:.1}%", uplift.winrate_with_pct()); - let winrate_delta = - format!("{:+.1} pts", uplift.winrate_with_pct() - uplift.winrate_without_pct()); - let wins_delta = format!("{extra_wins:+}"); - let profit_delta = - format!("{}{}", if extra_profit >= 0.0 { "+" } else { "-" }, fmt_usd(extra_profit.abs())); - format!( - "\ - \ - \ - \ - \ - \ - \ - \ - \ - \ -
without PropAMMwith PropAMMdifference
win rate{}{}{}
orders won{}{}{}
quoted output vs settled{}{}{} ({})
fee captured for LPs{}{}{}
\ -

Same {} orders, same block states — the mock's presence is the only \ - difference. The first three rows are the taker's side and barely move by design: the router \ - pins a surplus quote's output to the public reference, so the underbid is not passed on. \ - It lands in the last row instead, on {} of flow the pool took.

", - winrate_without, - winrate_with, - delta( - sign_of(uplift.winrate_with_pct() - uplift.winrate_without_pct()), - &winrate_delta, + "
\ +
hindsight report {scope}\ + mock PropAMM {verdict}\ + {}
\ +
{}{}
\ +
{}
\ +
{}{}{}{}{}
\ +
{}", + escape(pair), + world_column( + "without PropAMM", + "", + uplift.profit_without_usd, + uplift.winrate_without_pct(), + uplift.median_bps_without, + None ), - uplift.wins_without, - uplift.wins_with, - delta(extra_wins.cmp(&0), &wins_delta), - fmt_usd(uplift.profit_without_usd), - fmt_usd(uplift.profit_with_usd), - delta(sign_of(extra_profit), &profit_delta), - escape(&profit_pct), - fmt_usd(0.0), - fmt_usd(propamm.fee_headroom_usd), - delta( - sign_of(propamm.fee_headroom_usd), - &format!("+{}", fmt_usd(propamm.fee_headroom_usd)), + world_column( + "with PropAMM", + "on", + uplift.profit_with_usd, + uplift.winrate_with_pct(), + uplift.median_bps_with, + Some(( + uplift.extra_profit_usd(), + uplift.winrate_with_pct() - uplift.winrate_without_pct() + )) ), - uplift.orders, - fmt_usd(propamm.captured_flow_usd), + lp_capture(propamm), + mini(&fmt_count(summary.distinct_blocks), "blocks"), + mini(&fmt_count(uplift.orders), "orders scored both ways"), + mini(&format!("{:+}", uplift.extra_wins()), "extra orders won"), + mini(&fmt_count(summary.total), "comparisons"), + mini(&fmt_usd(propamm.captured_flow_usd), "flow through the pool"), + propamm_groups(&propamm.groups), ) } -/// The run's totals under the verdict: how much flow the pool took and what fee it took on it. -/// -/// Kept small and beneath the verdict — these are the size of the result, not the result. -fn propamm_totals(propamm: &PropAmm) -> String { - let stat = |value: &str, label: &str| { +/// One world's three headline figures, with the deltas attached to the second column. +fn world_column( + title: &str, + modifier: &str, + profit_usd: f64, + winrate_pct: f64, + median_bps: Option, + deltas: Option<(f64, f64)>, +) -> String { + let (profit_delta, winrate_delta) = deltas.map_or_else( + || (String::new(), String::new()), + |(profit, winrate)| { + ( + signed_delta(profit, &fmt_usd_signed(profit)), + signed_delta(winrate, &format!("{winrate:+.1} pts")), + ) + }, + ); + let big = |value: &str, label: &str, delta: &str| { format!( - "
{value}
\ -
{label}
" + "
{value}
\ +
{label} {delta}
" ) }; format!( - "
{}{}{}{}{}
", - stat(&fmt_count(propamm.won), "orders won"), - stat(&pct(propamm.won, propamm.solved), "of orders solved"), - stat(&fmt_usd(propamm.captured_flow_usd), "flow captured"), - stat(&fmt_bps(propamm.median_headroom_bps), "median fee headroom"), - stat(&fmt_bps(propamm.avg_headroom_bps()), "fee headroom, flow-weighted"), + "
{}
{}{}{}
", + escape(title), + big(&fmt_usd(profit_usd), "Fynd savings (wins uplift)", &profit_delta), + big(&format!("{winrate_pct:.1}%"), "win rate", &winrate_delta), + big(&fmt_bps_signed(median_bps), "median savings bps (wins)", ""), ) } +/// The fee the pool captured for its LPs — where the underbid lands, since the taker's quote is +/// pinned to the public reference. +fn lp_capture(propamm: &PropAmm) -> String { + format!( + "
+{}
\ +
captured for LPs {}
", + fmt_usd(propamm.fee_headroom_usd), + signed_delta(1.0, &fmt_bps(propamm.avg_headroom_bps())), + ) +} + +/// A signed figure coloured by direction. Colour is never the only carrier — the sign is in the +/// text. +fn signed_delta(value: f64, formatted: &str) -> String { + let cls = match value + .partial_cmp(&0.0) + .unwrap_or(std::cmp::Ordering::Equal) + { + std::cmp::Ordering::Greater => "pos", + std::cmp::Ordering::Less => "neg", + std::cmp::Ordering::Equal => "idlenum", + }; + format!("{}", escape(formatted)) +} + +/// A USD amount with an explicit sign, for a delta. +fn fmt_usd_signed(value: f64) -> String { + format!("{}{}", if value < 0.0 { "-" } else { "+" }, fmt_usd(value.abs())) +} + /// The offset groups, one card each, ascending by price. /// /// Each card's big number is the count the group's expectation is about: for a below-market group @@ -492,40 +474,6 @@ fn share_pct_of(pct: f64) -> String { format!("{pct:.0}%") } -/// Per-order-direction breakdown. A row's pair can differ from the mirrored pair: the mock serves -/// any route with a leg it can price, so a `DAI→USDC` order routed through `WETH` shows up here -/// too. -fn propamm_pair_table(pairs: &[PropAmmPair]) -> String { - if pairs.is_empty() { - return "

no orders

".to_string(); - } - let mut table = String::from( - "\ - \ - ", - ); - for pair in pairs { - let _ = write!( - table, - "\ - \ - ", - escape(&pair.token_in), - escape(&pair.token_out), - escape(&short_hash(&pair.token_in)), - escape(&short_hash(&pair.token_out)), - pair.solved, - pair.won, - pct(pair.won, pair.solved), - fmt_usd(pair.captured_flow_usd), - fmt_usd(pair.fee_headroom_usd), - fmt_bps(pair.median_headroom_bps), - ); - } - table.push_str("
order pairsolvedwinswin%flow capturedfee headroommedian headroom bps
{} → {}{}{}{}{}{}{}
"); - table -} - fn section(title: &str, body: &str) -> String { format!("

{}

{body}
", escape(title)) } @@ -661,9 +609,17 @@ section { background: #211a30; border: 1px solid #362b4a; border-radius: 8px; /* A section's framing sentence: what the numbers below mean, before they are read. */ .note { color: #b9adcf; margin: 0 0 .5rem; max-width: 68ch; line-height: 1.55; } /* The run's verdict, sized like the report's headline savings figure. */ -.propammhead { display: flex; flex-wrap: wrap; align-items: baseline; gap: 2rem; } -.propammsub { color: #b9adcf; font-size: .9rem; max-width: 52ch; line-height: 1.5; } .idlenum { color: #9a8bbf; } +/* The two worlds side by side, so the comparison is read by scanning across. */ +.worlds { display: flex; flex-wrap: wrap; gap: 1.5rem; margin-top: 1.5rem; } +.world { flex: 1 1 20rem; border: 1px solid #2f2540; border-radius: 8px; padding: 1.1rem 1.3rem; } +/* The "with" column is the answer, so it is the one that is lit. */ +.world.on { border-color: #43a047; background: #16241a; } +.worldtitle { color: #9a8bbf; font-size: .78rem; text-transform: uppercase; letter-spacing: .06em; + margin-bottom: .75rem; } +.world .herostat { margin-bottom: .9rem; } +.world .herostat:last-child { margin-bottom: 0; } +.lprow { margin-top: 1.5rem; } /* One card per offset group. They wrap rather than scroll, so a wider ladder stays readable. */ .groups { display: flex; flex-wrap: wrap; gap: 1rem; margin: 1.5rem 0; } .group { flex: 1 1 15rem; border: 1px solid #2f2540; border-radius: 6px; padding: .9rem 1rem; } @@ -842,73 +798,25 @@ mod tests { fn test_escape_replaces_markup() { assert_eq!(escape("&\"x\""), "<a>&"x""); } - - /// A report from records carrying mock-`PropAMM` outcomes. - fn propamm_report(won: bool) -> Report { - let records: Vec = vec![serde_json::json!({ - "block": 1, - "settled_tx": "0xabc0000000000000000000000000000000000000000000000000000000000001", - "venue": "relay", "solver": "1inch", - "token_in": "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2", - "token_out": "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", - "top": {"verdict": "win", "net_bps": 20.0, "improvement_usd": 12.0, "settled_value_usd": 1000.0}, - "propamm": { - "pair": "WETH/USDC", "won": won, - "fee_headroom_bps": won.then_some(4.0), - "committed_usd": won.then_some(1_000.0), - "fee_headroom_usd": won.then_some(0.4), - }, - })] - .into_iter() - .map(|v| serde_json::from_value(v).unwrap()) - .collect(); - build(&records) - } - - #[test] - fn test_propamm_section_omitted_without_the_harness() { - // An ordinary run's report must look exactly as it did before the harness existed. - assert!(!render(&sample_report(), None).contains("Mock PropAMM")); - } - - #[test] - fn test_propamm_section_names_the_mirrored_pair_and_its_numbers() { - let html = render(&propamm_report(true), None); - assert!(html.contains("Mock PropAMM (exclusive route)")); - assert!(html.contains("WETH/USDC"), "the section must name the pool it stood in for"); - assert!(html.contains("median fee headroom")); - // The order pair is rendered short, with the full addresses in the hover title. - assert!(html.contains("0xc02aaa39…756cc2")); - assert!(html.contains("0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2")); - } - - #[test] - fn test_propamm_section_renders_a_run_the_pool_never_won() { - // A zero-win run is a result, not an error: the section still renders, with an em dash - // where the median headroom is undefined. - let html = render(&propamm_report(false), None); - assert!(html.contains("Mock PropAMM (exclusive route)")); - assert!(html.contains("0.0%"), "a zero winrate must be stated, not omitted"); - } - - /// A report from calibrated records across the three offset groups. + /// A report from calibrated records across the three offset groups, with the A/B attached. fn group_report(above_fee_bps: f64) -> Report { let record = |block: u64, offset: i32, won: bool, fee: f64| { serde_json::json!({ "block": block, "settled_tx": format!("0x{block:064x}"), "venue": "relay", "solver": "1inch", - "token_in": "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2", - "token_out": "0xdac17f958d2ee523a2206206994597c13d831ec7", - "top": {"verdict": "win", "net_bps": 5.0, "settled_value_usd": 1000.0}, + "token_in": "0x0000000000000000000000000000000000000000", + "token_out": "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", + "top": {"verdict": "win", "net_bps": 5.0, "improvement_usd": 4.0, + "settled_value_usd": 1000.0}, "propamm": { - "pair": "WETH/USDT", "offset_bps": offset, "won": won, + "pair": "ETH/USDC", "offset_bps": offset, "won": won, "fee_headroom_bps": won.then_some(fee), "committed_usd": won.then_some(1_000.0), "fee_headroom_usd": won.then_some(fee / 10.0), - "without_won": false, "with_won": won, - "without_improvement_usd": -1.0, - "with_improvement_usd": if won { 4.0 } else { -1.0 }, + "without_won": true, "with_won": true, + "without_improvement_usd": 4.0, "with_improvement_usd": 4.0, + "without_net_bps": 5.0, "with_net_bps": 5.0, }, }) }; @@ -923,66 +831,78 @@ mod tests { build(&records) } - #[test] - fn test_propamm_section_leads_the_document() { - // A calibrated run exists to answer one question, so its verdict comes before the savings - // headline rather than three sections down. - let html = render(&group_report(5.0), None); - let propamm = html - .find("Mock PropAMM") - .expect("section present"); - let savings = html - .find("Fynd savings") - .expect("hero present"); - assert!(propamm < savings, "the PropAMM verdict must precede the savings headline"); + /// A calibrated run whose records carry no offset, so no group reaches a conclusion. + fn uncalibrated_report() -> Report { + let records: Vec = vec![serde_json::json!({ + "block": 1, + "settled_tx": "0xabc0000000000000000000000000000000000000000000000000000000000001", + "venue": "relay", "solver": "1inch", + "token_in": "0xaaa", "token_out": "0xbbb", + "top": {"verdict": "win", "net_bps": 20.0, "improvement_usd": 12.0, + "settled_value_usd": 1000.0}, + "propamm": {"pair": "ETH/USDC", "won": false}, + })] + .into_iter() + .map(|v| serde_json::from_value(v).unwrap()) + .collect(); + build(&records) } #[test] - fn test_overall_verdict_is_a_single_word() { - assert!(render(&group_report(5.0), None).contains(">PASS<")); - assert!(render(&group_report(20.0), None).contains(">FAIL<")); + fn test_ordinary_run_keeps_the_single_world_hero() { + // A run without the harness must look exactly as it did before it existed. + let html = render(&sample_report(), None); + assert!(html.contains("Fynd savings")); + assert!(!html.contains("without PropAMM")); + assert!(!html.contains("mock PropAMM")); } #[test] - fn test_group_cards_lead_with_the_verdict_and_name_each_price() { + fn test_calibrated_run_splits_the_hero_into_both_worlds() { + // The point of the redesign: the headline figures appear twice, once per world, rather than + // one number that silently mixes them. let html = render(&group_report(5.0), None); - assert!(html.contains("-5 bps (below market)")); - assert!(html.contains("at market")); - assert!(html.contains("+5 bps (above market)")); - // The verdict is a word, never colour alone. - assert!(html.contains(">pass<"), "a passing group must say so in text"); - assert!(html.contains("never selected")); - assert!(html.contains("selected only on gas, zero fee")); + assert!(html.contains("without PropAMM")); + assert!(html.contains("with PropAMM")); + assert_eq!( + html.matches("Fynd savings (wins uplift)") + .count(), + 2, + "the savings figure is stated for each world" + ); + assert_eq!( + html.matches("herolab\">win rate") + .count(), + 2 + ); + assert_eq!( + html.matches("median savings bps (wins)") + .count(), + 2 + ); } #[test] - fn test_a_failing_group_renders_the_reason_not_just_a_colour() { - let html = render(&group_report(20.0), None); - assert!(html.contains(">FAIL<")); - assert!( - html.contains("took a fee of 20.00 bps"), - "the reason must be readable without opening the JSONL" - ); + fn test_split_hero_replaces_rather_than_joins_the_single_world_one() { + // Two heroes would mean two different win rates on one page, one of them ambiguous. + let html = render(&group_report(5.0), None); + assert_eq!(html.matches("class=\"hero\"").count(), 1); } #[test] - fn test_group_panel_says_so_when_nothing_was_calibrated() { - // A run whose settled trades never touched the mirrored pair must explain the empty panel - // rather than render three blank cards. - let html = render(&propamm_report(true), None); - assert!(html.contains("No calibrated orders")); + fn test_hero_names_the_pair_and_the_overall_verdict() { + let html = render(&group_report(5.0), None); + assert!(html.contains("ETH/USDC"), "the pair is named by symbol, not by address"); + assert!(html.contains("mock PropAMM PASS")); + assert!(render(&group_report(20.0), None).contains("mock PropAMM FAIL")); } #[test] - fn test_uplift_table_shows_both_worlds_and_the_difference() { + fn test_hero_shows_where_the_underbid_lands() { + // The taker-side figures barely move by design, so the LP capture has to be on the page or + // the run reads as a null result. let html = render(&group_report(5.0), None); - assert!(html.contains("without PropAMM")); - assert!(html.contains("with PropAMM")); - assert!(html.contains("win rate")); - assert!(html.contains("quoted output vs settled")); - assert!(html.contains("fee captured for LPs")); - // The difference must be signed, so a regression reads as one. - assert!(html.contains("delta"), "each row carries a signed difference"); + assert!(html.contains("captured for LPs")); } #[test] @@ -996,8 +916,29 @@ mod tests { } #[test] - fn test_uplift_table_says_so_when_nothing_was_scored_in_both_worlds() { - let html = render(&propamm_report(true), None); - assert!(html.contains("nothing to"), "an empty A/B must explain itself"); + fn test_group_cards_name_each_price_and_its_expectation() { + let html = render(&group_report(5.0), None); + assert!(html.contains("-5 bps (below market)")); + assert!(html.contains("at market")); + assert!(html.contains("+5 bps (above market)")); + assert!(html.contains("never selected")); + assert!(html.contains("selected only on gas, zero fee")); + } + + #[test] + fn test_a_failing_group_renders_the_reason_not_just_a_colour() { + let html = render(&group_report(20.0), None); + assert!(html.contains(">FAIL<")); + assert!( + html.contains("took a fee of 20.00 bps"), + "the reason must be readable without opening the JSONL" + ); + } + + #[test] + fn test_group_panel_says_so_when_nothing_was_calibrated() { + // A run whose settled trades never touched the mirrored pair must explain the empty panel + // rather than render three blank cards. + assert!(render(&uncalibrated_report(), None).contains("No calibrated orders")); } } diff --git a/tools/hindsight/src/report/record.rs b/tools/hindsight/src/report/record.rs index 2727a2b2..4d46047a 100644 --- a/tools/hindsight/src/report/record.rs +++ b/tools/hindsight/src/report/record.rs @@ -62,6 +62,12 @@ pub(crate) struct PropAmm { /// USD Fynd gained over the settled trade with the mock available. #[serde(default)] pub with_improvement_usd: Option, + /// Net-of-gas bps over the settled trade without the mock. + #[serde(default)] + pub without_net_bps: Option, + /// Net-of-gas bps over the settled trade with the mock available. + #[serde(default)] + pub with_net_bps: Option, } /// Fynd's result at one block state. @@ -195,6 +201,8 @@ mod tests { with_won: Some(true), without_improvement_usd: Some(-1.0), with_improvement_usd: Some(3.0), + without_net_bps: Some(-8.0), + with_net_bps: Some(24.0), }, ); diff --git a/tools/hindsight/src/resolve/monitor.rs b/tools/hindsight/src/resolve/monitor.rs index 11e9c023..fea3f5f6 100644 --- a/tools/hindsight/src/resolve/monitor.rs +++ b/tools/hindsight/src/resolve/monitor.rs @@ -993,11 +993,22 @@ fn score_both_worlds( biguint_to_u256_opt(amount_out) .and_then(|out| prices.savings_usd(range.token_out, out, range.settled_amount_out)) }; + // A win is gross output strictly above the settled gross output — the basis `compare::verdict` + // uses, so these figures drop straight into the report's headline. The settled route's gas is + // often unattributable, which makes gross-vs-gross the only always-like-for-like comparison. + let settled_out = u256_to_biguint(range.settled_amount_out); + let with_won = range.top.verdict == super::compare::Verdict::Win; propamm::report::AbResult { - without_won: Some(without.amount_out_net_gas > settled_net_gas), - with_won: with.map(|s| u256_to_biguint(s.amount_out_net_gas) > settled_net_gas), + without_won: Some(without.amount_out > settled_out), + with_won: with.map(|_| with_won), without_improvement_usd: improvement(&without.amount_out), with_improvement_usd: with.and_then(|s| improvement(&u256_to_biguint(s.amount_out))), + // Net of each side's own gas, matching the headline's median savings figure. + without_net_bps: fynd_tools_common::bps::raw_bps_diff( + &without.amount_out_net_gas, + &settled_net_gas, + ), + with_net_bps: range.top.deltas.net_bps, } } From a367d8330e1e99f7b4595910ba1d76250d892ad9 Mon Sep 17 00:00:00 2001 From: TAMARA LIPOWSKI Date: Tue, 28 Jul 2026 21:38:25 -0400 Subject: [PATCH 11/12] feat(hindsight): lead the report with the three price tests Names each test by the pool's price against the best public route rather than by a basis-point offset, states the rule and what happened as two sentences beside each other, and puts one TESTS PASSED verdict above the three cards. --- tools/hindsight/src/report/aggregate.rs | 91 +++++++++++++++--- tools/hindsight/src/report/html.rs | 120 ++++++++++++++---------- 2 files changed, 148 insertions(+), 63 deletions(-) diff --git a/tools/hindsight/src/report/aggregate.rs b/tools/hindsight/src/report/aggregate.rs index 46532b44..d74722b7 100644 --- a/tools/hindsight/src/report/aggregate.rs +++ b/tools/hindsight/src/report/aggregate.rs @@ -137,6 +137,16 @@ impl PropAmm { GroupVerdict::NoData } } + + /// Groups that reached a conclusion, over the groups that were run. + pub(crate) fn conclusive(&self) -> (usize, usize) { + let decided = self + .groups + .iter() + .filter(|group| group.verdict != GroupVerdict::NoData) + .count(); + (decided, self.groups.len()) + } } /// Whether an offset group behaved as its price implies. @@ -178,23 +188,48 @@ pub(crate) struct PropAmmGroup { } impl PropAmmGroup { - /// The expectation this group's price implies, as a short phrase for the report. - pub(crate) fn expectation(&self) -> &'static str { + /// What this group is testing, in terms of the pool's price against the best public route. + /// + /// Named by the relationship rather than the raw offset: "worse than the best route" is the + /// thing being asserted, and the basis-point figure is only how it was arranged. + pub(crate) fn title(&self) -> &'static str { match self.offset_bps.cmp(&0) { - std::cmp::Ordering::Less => "never selected", - std::cmp::Ordering::Equal => "selected only on gas, zero fee", - std::cmp::Ordering::Greater => "selected, fee at most the offset", + std::cmp::Ordering::Less => "Priced worse than the best route", + std::cmp::Ordering::Equal => "Priced equal to the best route", + std::cmp::Ordering::Greater => "Priced better than the best route", } } - /// Share of the group's orders the mock was selected for, as a percentage. - // Precision loss is irrelevant: these are order counts, far below f64's exact-integer range. - #[expect(clippy::cast_precision_loss)] - pub(crate) fn selected_pct(&self) -> f64 { + /// The expectation this group's price implies, as one plain sentence. + pub(crate) fn expectation(&self) -> String { + match self.offset_bps.cmp(&0) { + std::cmp::Ordering::Less => "Must never be chosen.".to_string(), + std::cmp::Ordering::Equal => { + "Can only be chosen for its cheaper gas, and then charges no fee.".to_string() + } + std::cmp::Ordering::Greater => format!( + "Should be chosen, and cannot charge more than the {} bps gap.", + self.offset_bps + ), + } + } + + /// What actually happened, as one plain sentence to sit beside the expectation. + pub(crate) fn outcome(&self) -> String { if self.orders == 0 { - return 0.0; + return "No orders landed in this group.".to_string(); + } + if self.selected == 0 { + return format!("Never chosen, across {} orders.", self.orders); + } + match (self.median_fee_bps, self.max_fee_bps) { + (Some(median), Some(max)) => format!( + "Chosen for {} of {} orders, charging {median:.2} bps typically and {max:.2} bps at \ + most.", + self.selected, self.orders + ), + _ => format!("Chosen for {} of {} orders.", self.selected, self.orders), } - self.selected as f64 / self.orders as f64 * 100.0 } } @@ -951,7 +986,8 @@ mod tests { } #[test] - fn test_selected_pct_is_zero_rather_than_nan_when_empty() { + fn test_empty_group_says_so_rather_than_claiming_it_was_never_chosen() { + // "Never chosen" and "no orders" are different findings; the card must not conflate them. let group = PropAmmGroup { offset_bps: 0, orders: 0, @@ -960,7 +996,36 @@ mod tests { median_fee_bps: None, verdict: GroupVerdict::NoData, }; - assert!(group.selected_pct().abs() < f64::EPSILON); + assert!(group.outcome().contains("No orders")); + } + + #[test] + fn test_each_group_states_its_rule_in_terms_of_the_best_route() { + // The card's job is to say what is being tested without the reader translating basis + // points. + let group = |offset_bps: i32| PropAmmGroup { + offset_bps, + orders: 4, + selected: 0, + max_fee_bps: None, + median_fee_bps: None, + verdict: GroupVerdict::Pass, + }; + assert_eq!(group(-5).title(), "Priced worse than the best route"); + assert_eq!(group(0).title(), "Priced equal to the best route"); + assert_eq!(group(5).title(), "Priced better than the best route"); + assert!(group(-5) + .expectation() + .contains("never be chosen")); + assert!(group(0) + .expectation() + .contains("no fee")); + assert!(group(5) + .expectation() + .contains("5 bps gap")); + assert!(group(-5) + .outcome() + .contains("Never chosen, across 4 orders")); } /// A calibrated record carrying the with/without A/B. diff --git a/tools/hindsight/src/report/html.rs b/tools/hindsight/src/report/html.rs index 9b1f3efe..c264c76a 100644 --- a/tools/hindsight/src/report/html.rs +++ b/tools/hindsight/src/report/html.rs @@ -306,11 +306,13 @@ fn propamm_hero(propamm: &PropAmm, summary: &Summary, filter: Option<&str>) -> S
hindsight report {scope}\ mock PropAMM {verdict}\ {}
\ + {}\
{}{}
\
{}
\
{}{}{}{}{}
\ - {}", + ", escape(pair), + propamm_tests(propamm), world_column( "without PropAMM", "", @@ -336,6 +338,26 @@ fn propamm_hero(propamm: &PropAmm, summary: &Summary, filter: Option<&str>) -> S mini(&format!("{:+}", uplift.extra_wins()), "extra orders won"), mini(&fmt_count(summary.total), "comparisons"), mini(&fmt_usd(propamm.captured_flow_usd), "flow through the pool"), + ) +} + +/// The three price tests: one banner verdict, then one card per test. +/// +/// Each card leads with what it is testing — the pool's price against the best public route — then +/// states the rule and what happened, so a reader never has to translate a basis-point offset into +/// an expectation themselves. +fn propamm_tests(propamm: &PropAmm) -> String { + let (decided, total) = propamm.conclusive(); + let (banner, cls) = match propamm.verdict() { + GroupVerdict::Pass => ("TESTS PASSED", "pos"), + GroupVerdict::Fail(_) => ("TESTS FAILED", "neg"), + GroupVerdict::NoData => ("NO CONCLUSION YET", "idlenum"), + }; + format!( + "
\ +
{banner}
\ +
{decided} of {total} price tests conclusive
\ +
{}", propamm_groups(&propamm.groups), ) } @@ -403,23 +425,22 @@ fn fmt_usd_signed(value: f64) -> String { format!("{}{}", if value < 0.0 { "-" } else { "+" }, fmt_usd(value.abs())) } -/// The offset groups, one card each, ascending by price. +/// One card per price test, ascending by price. /// -/// Each card's big number is the count the group's expectation is about: for a below-market group -/// that is how many times it was wrongly selected (so zero is the good answer), and above market it -/// is how many times it won. The number is coloured by outcome and always sits beside a word. +/// The verdict word is the largest thing on the card, because that is the answer. Under it the rule +/// and the observation sit as two plain sentences, so "pass" is always accompanied by what passed. fn propamm_groups(groups: &[PropAmmGroup]) -> String { if groups.is_empty() { - return "

No calibrated orders — no settled trade in this run was on \ - the mirrored pair.

" + return "

No calibrated orders yet — no settled trade so far was on the \ + mirrored pair.

" .to_string(); } let mut cards = String::new(); - for group in groups { + for (index, group) in groups.iter().enumerate() { let (word, cls) = match &group.verdict { - GroupVerdict::Pass => ("pass", "pos"), + GroupVerdict::Pass => ("PASS", "pos"), GroupVerdict::Fail(_) => ("FAIL", "neg"), - GroupVerdict::NoData => ("no data", "idlenum"), + GroupVerdict::NoData => ("NO DATA", "idlenum"), }; let detail = match &group.verdict { GroupVerdict::Fail(reason) => { @@ -429,51 +450,32 @@ fn propamm_groups(groups: &[PropAmmGroup]) -> String { }; let _ = write!( cards, - "
\ -
{}
\ + "
\ +
{}. {}
\ +
{}
\
{word}
\ -
{}
\ -
{} ({})
\ -
fee taken {} median, {} max, \ - of at most {}
\ -
{}
{detail}\ +
{}
\ +
{}
{detail}\
", + index + 1, + escape(group.title()), escape(&fmt_offset(group.offset_bps)), - group.selected, - escape(&format!("selected of {} orders", group.orders)), - escape(&share_pct_of(group.selected_pct())), - escape(&fmt_bps(group.median_fee_bps)), - escape(&fmt_bps(group.max_fee_bps)), - escape(&fmt_offset_ceiling(group.offset_bps)), - escape(group.expectation()), + escape(&group.expectation()), + escape(&group.outcome()), ); } format!("
{cards}
") } -/// The fee ceiling a group's price implies, for the "of at most" caption. -fn fmt_offset_ceiling(offset_bps: i32) -> String { - if offset_bps <= 0 { - "0.0 bps".to_string() - } else { - format!("{offset_bps}.0 bps") - } -} - /// An offset as a signed bps label, so a column header reads as a price and not a bare number. fn fmt_offset(offset_bps: i32) -> String { match offset_bps.cmp(&0) { - std::cmp::Ordering::Less => format!("{offset_bps} bps (below market)"), - std::cmp::Ordering::Equal => "at market".to_string(), - std::cmp::Ordering::Greater => format!("+{offset_bps} bps (above market)"), + std::cmp::Ordering::Less => format!("set {} bps below it", -offset_bps), + std::cmp::Ordering::Equal => "set exactly on it".to_string(), + std::cmp::Ordering::Greater => format!("set {offset_bps} bps above it"), } } -/// A percentage that is already computed, rendered like the report's other shares. -fn share_pct_of(pct: f64) -> String { - format!("{pct:.0}%") -} - fn section(title: &str, body: &str) -> String { format!("

{}

{body}
", escape(title)) } @@ -623,8 +625,15 @@ section { background: #211a30; border: 1px solid #362b4a; border-radius: 8px; /* One card per offset group. They wrap rather than scroll, so a wider ladder stays readable. */ .groups { display: flex; flex-wrap: wrap; gap: 1rem; margin: 1.5rem 0; } .group { flex: 1 1 15rem; border: 1px solid #2f2540; border-radius: 6px; padding: .9rem 1rem; } -.grouphead { display: flex; align-items: center; justify-content: space-between; gap: .75rem; } -.groupoff { font-weight: 600; font-size: .9rem; } +.grouphead { font-weight: 600; font-size: .95rem; } +.groupoff { color: #9a8bbf; font-size: .78rem; margin-top: .15rem; } +.grouprule { color: #b9adcf; font-size: .82rem; line-height: 1.45; } +.groupsaw { color: #e9e2f5; font-size: .82rem; line-height: 1.45; margin-top: .35rem; font-weight: 600; } +/* A card's border echoes its verdict, but the verdict word above it is what carries the meaning. */ +.group.poscard { border-color: #43a047; background: #16241a; } +.group.negcard { border-color: #e53935; background: #241616; } +/* The tests banner: the single answer, above the three cards that justify it. */ +.testsbanner { margin-top: 1.75rem; } /* The number the group's expectation is about, big enough to read across a room. */ .groupbig { font-size: 2.6rem; font-weight: 700; line-height: 1.05; margin-top: .5rem; } .groupcap { color: #9a8bbf; font-size: .78rem; } @@ -636,7 +645,7 @@ section { background: #211a30; border: 1px solid #362b4a; border-radius: 8px; .chip.idlenumchip { background: #2f2540; color: #9a8bbf; } .groupfail { color: #ef9a9a; font-size: .78rem; margin: .6rem 0 0; line-height: 1.45; } /* Each sub-test carries its own verdict word, sized so the three read at a glance together. */ -.groupverdict { font-size: 1.5rem; font-weight: 700; letter-spacing: .04em; margin-top: .45rem; } +.groupverdict { font-size: 2.4rem; font-weight: 800; letter-spacing: .04em; margin: .5rem 0 .6rem; } /* The with/without table: the "with" column is the answer, so it carries the weight. */ .ab { margin: 1.5rem 0 .25rem; } .ab th { color: #9a8bbf; font-weight: 500; } @@ -916,13 +925,24 @@ mod tests { } #[test] - fn test_group_cards_name_each_price_and_its_expectation() { + fn test_group_cards_name_each_test_in_terms_of_the_best_route() { + // A reader should not have to turn a basis-point offset into an expectation themselves. let html = render(&group_report(5.0), None); - assert!(html.contains("-5 bps (below market)")); - assert!(html.contains("at market")); - assert!(html.contains("+5 bps (above market)")); - assert!(html.contains("never selected")); - assert!(html.contains("selected only on gas, zero fee")); + assert!(html.contains("Priced worse than the best route")); + assert!(html.contains("Priced equal to the best route")); + assert!(html.contains("Priced better than the best route")); + assert!(html.contains("Must never be chosen.")); + assert!(html.contains("charges no fee")); + assert!(html.contains("cannot charge more than the 5 bps gap")); + // And each card says what actually happened next to what should have. + assert!(html.contains("Never chosen, across")); + } + + #[test] + fn test_tests_banner_states_one_answer_for_the_three_cards() { + assert!(render(&group_report(5.0), None).contains("TESTS PASSED")); + assert!(render(&group_report(20.0), None).contains("TESTS FAILED")); + assert!(render(&group_report(5.0), None).contains("price tests conclusive")); } #[test] From 3e36c0621e8fada492aab15992a8fbc3766d0080 Mon Sep 17 00:00:00 2001 From: TAMARA LIPOWSKI Date: Tue, 28 Jul 2026 22:20:29 -0400 Subject: [PATCH 12/12] fix(hindsight): score the with/without A/B on the same basis The with side read the trade's verdict, which excludes sandwiched trades, while the without side compared amounts directly. One sandwiched order therefore counted as a win in one world and a loss in the other, showing a win-rate drop and a $3.79 loss that neither world had. Restrict the comparison to scored trades, matching the savings headline it sits beside. --- tools/hindsight/src/report/aggregate.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/hindsight/src/report/aggregate.rs b/tools/hindsight/src/report/aggregate.rs index d74722b7..833c073e 100644 --- a/tools/hindsight/src/report/aggregate.rs +++ b/tools/hindsight/src/report/aggregate.rs @@ -347,6 +347,12 @@ fn propamm(records: &[Comparison]) -> Option { /// Sums the with/without A/B over the orders that carry both sides. /// +/// Restricted to **scored** trades — win or loss — which is the same basis the report's own savings +/// headline uses. A sandwiched trade is excluded there because MEV moved the settled output it is +/// measured against, and including it here would compare a sandwich-aware verdict on one side +/// against a plain amount comparison on the other, so one order can appear to flip when both worlds +/// in fact produced the same result. +/// /// Wins and profits are counted independently: an order can be scored for its verdict but have an /// unpriced output token, so it contributes to the win counts and not to the USD. fn uplift(scoped: &[&Comparison]) -> Uplift { @@ -355,6 +361,7 @@ fn uplift(scoped: &[&Comparison]) -> Uplift { let mut bps_with: Vec = Vec::new(); for propamm in scoped .iter() + .filter(|r| r.top.is_scored()) .filter_map(|r| r.propamm.as_ref()) { let (Some(without_won), Some(with_won)) = (propamm.without_won, propamm.with_won) else {