ctutils: add primitives::borrowing_sub_mask_u32/u64 (#1577)

It seems `crypto-bigint` has undergone a bit of a performance regression
and these are the primitive needed to fix it.

Author: tarcieri

ctutils/CHANGELOG.md

@@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+### Added
+- `primitives::borrowing_sub_mask_u32` and `borrowing_sub_mask_u64` for propagating
+  word-sized borrow masks without per-limb `Choice` conversions
+
 ## 0.4.5 (2026-10-23)
 ### Added
 - `primitives` module ([#1570], [#1572])

ctutils/checkct/driver/src/driver/primitives.rs

@@ -145,3 +145,72 @@ macro_rules! subtraction_checks {
 
 subtraction_checks!(u32_sub, u32, borrowing_sub_u32, to_u32_mask);
 subtraction_checks!(u64_sub, u64, borrowing_sub_u64, to_u64_mask);
+
+macro_rules! subtraction_mask_checks {
+    ($module:ident, $ty:ty, $sub:ident) => {
+        mod $module {
+            use super::*;
+
+            #[checkct]
+            pub fn borrowing_sub_mask() {
+                black_box(PublicRng.next_u32());
+                black_box(primitives::$sub(
+                    private_integer!($ty),
+                    private_integer!($ty),
+                    private_integer!($ty),
+                ));
+            }
+
+            #[checkct]
+            pub fn borrowing_sub_mask_zero_borrow() {
+                black_box(PublicRng.next_u32());
+                black_box(primitives::$sub(
+                    private_integer!($ty),
+                    private_integer!($ty),
+                    0,
+                ));
+            }
+
+            #[checkct]
+            pub fn borrowing_sub_mask_limbs() {
+                black_box(PublicRng.next_u32());
+                let lhs = [private_integer!($ty), private_integer!($ty)];
+                let rhs = [private_integer!($ty), private_integer!($ty)];
+                let mut borrow = private_integer!($ty);
+                let mut difference = [0; 2];
+                for i in 0..2 {
+                    (difference[i], borrow) = primitives::$sub(lhs[i], rhs[i], borrow);
+                }
+                black_box((difference, borrow));
+            }
+
+            /// Discarding the final borrow can change how the subtraction is optimized.
+            #[checkct]
+            pub fn wrapping_sub_mask_limbs() {
+                black_box(PublicRng.next_u32());
+                let lhs = [private_integer!($ty), private_integer!($ty)];
+                let rhs = [private_integer!($ty), private_integer!($ty)];
+                let mut borrow = 0;
+                let mut difference = [0; 2];
+                for i in 0..2 {
+                    (difference[i], borrow) = primitives::$sub(lhs[i], rhs[i], borrow);
+                }
+                black_box(difference);
+            }
+
+            /// Keep the subtraction and masked add-back together to test their optimization.
+            #[checkct]
+            pub fn borrowing_sub_mask_add_back() {
+                black_box(PublicRng.next_u32());
+                let lhs = private_integer!($ty);
+                let rhs = private_integer!($ty);
+                let incoming = private_integer!($ty);
+                let (difference, borrow) = primitives::$sub(lhs, rhs, incoming);
+                black_box(difference.wrapping_add(rhs & borrow));
+            }
+        }
+    };
+}
+
+subtraction_mask_checks!(u32_sub_mask, u32, borrowing_sub_mask_u32);
+subtraction_mask_checks!(u64_sub_mask, u64, borrowing_sub_mask_u64);

ctutils/src/primitives.rs

@@ -89,27 +89,61 @@ comparison_primitives!(u32, le_u32, lt_u32, nz_u32, from_u32_lsb);
 comparison_primitives!(u64, le_u64, lt_u64, nz_u64, from_u64_lsb);
 
 macro_rules! borrowing_sub_primitives {
-    ($ty:ty, $sub:ident, $from_lsb:ident) => {
+    ($ty:ty, $sub:ident, $sub_mask:ident, $sub_inner:ident, $from_lsb:ident) => {
         /// Subtract `rhs` and an incoming borrow from `lhs`.
         ///
         /// Returns the wrapping difference and [`Choice::TRUE`] if the subtraction underflows.
         /// The incoming `borrow` subtracts one when true and zero when false.
         #[inline]
         #[must_use]
         pub const fn $sub(lhs: $ty, rhs: $ty, borrow: Choice) -> ($ty, Choice) {
-            let difference = lhs.wrapping_sub(rhs).wrapping_sub(borrow.to_u8() as $ty);
+            let (difference, borrow) = $sub_inner(lhs, rhs, borrow.to_u8() as $ty);
+            (difference, Choice::$from_lsb(borrow))
+        }
+
+        /// Subtract `rhs` and an incoming borrow from `lhs`, using word-sized borrow masks.
+        ///
+        /// The incoming borrow is the most significant bit of `borrow`; all other bits are ignored.
+        /// Returns the wrapping difference and an all-ones mask if the subtraction underflows,
+        /// or zero otherwise.
+        ///
+        /// The outgoing mask can be passed directly to the next limb's subtraction, avoiding
+        /// conversions to and from [`Choice`] at each step of a multi-limb subtraction.
+        #[inline]
+        #[must_use]
+        pub const fn $sub_mask(lhs: $ty, rhs: $ty, borrow: $ty) -> ($ty, $ty) {
+            let (difference, borrow) = $sub_inner(lhs, rhs, borrow >> (<$ty>::BITS - 1));
+            (difference, borrow.wrapping_neg())
+        }
+
+        // The incoming borrow must be zero or one; the outgoing borrow is also zero or one.
+        #[inline(always)]
+        const fn $sub_inner(lhs: $ty, rhs: $ty, borrow: $ty) -> ($ty, $ty) {
+            let difference = lhs.wrapping_sub(rhs).wrapping_sub(borrow);
             // Keep all arithmetic in the operand type: widening can trigger incorrect
             // optimization on older compilers: https://github.com/rust-lang/rust/issues/149522
-            // Keep the full-width bit opaque before narrowing it to Choice, so a caller's
-            // masked add-back cannot be folded into a secret-dependent branch.
+            // Keep the full-width bit opaque before forming a mask or narrowing it to Choice,
+            // so a caller's masked add-back cannot be folded into a secret-dependent branch.
             let borrow = black_box(bitborrow!(lhs, rhs, difference, <$ty>::BITS));
-            (difference, Choice::$from_lsb(borrow))
+            (difference, borrow)
         }
     };
 }
 
-borrowing_sub_primitives!(u32, borrowing_sub_u32, from_u32_lsb);
-borrowing_sub_primitives!(u64, borrowing_sub_u64, from_u64_lsb);
+borrowing_sub_primitives!(
+    u32,
+    borrowing_sub_u32,
+    borrowing_sub_mask_u32,
+    borrowing_sub_inner_u32,
+    from_u32_lsb
+);
+borrowing_sub_primitives!(
+    u64,
+    borrowing_sub_u64,
+    borrowing_sub_mask_u64,
+    borrowing_sub_inner_u64,
+    from_u64_lsb
+);
 
 /// Count leading zero bits, returning 32 if `value` is zero.
 #[inline]
@@ -276,6 +310,12 @@ mod tests {
         assert!(borrowing_sub_u64(0, 0, Choice::TRUE).1.to_bool());
         assert!(!borrowing_sub_u32(1, 0, Choice::TRUE).1.to_bool());
         assert!(!borrowing_sub_u64(1, 0, Choice::TRUE).1.to_bool());
+        assert!(borrowing_sub_mask_u32(0, 0, u32::MAX).0 == u32::MAX);
+        assert!(borrowing_sub_mask_u32(0, 0, u32::MAX).1 == u32::MAX);
+        assert!(borrowing_sub_mask_u64(0, 0, u64::MAX).0 == u64::MAX);
+        assert!(borrowing_sub_mask_u64(0, 0, u64::MAX).1 == u64::MAX);
+        assert!(borrowing_sub_mask_u32(1, 0, u32::MAX).1 == 0);
+        assert!(borrowing_sub_mask_u64(1, 0, u64::MAX).1 == 0);
         assert!(leading_zeros_u32(0) == 32);
         assert!(trailing_zeros_u32(0) == 32);
         assert!(leading_zeros_u64(0) == 64);
@@ -311,15 +351,17 @@ mod tests {
     }
 
     macro_rules! subtraction_tests {
-        ($test:ident, $ty:ty, $sub:ident) => {
+        ($test:ident, $ty:ty, $sub:ident, $sub_mask:ident) => {
             #[test]
             fn $test() {
                 let half = 1 << (<$ty>::BITS >> 1);
                 let high = 1 << (<$ty>::BITS - 1);
                 let values = [0, 1, half - 1, half, half + 1, high - 1, high, <$ty>::MAX];
                 for lhs in values {
                     for rhs in values {
-                        for incoming in [false, true] {
+                        // Include noncanonical masks: only the most significant bit matters.
+                        for mask in [0, 1, high - 1, high, <$ty>::MAX - 1, <$ty>::MAX] {
+                            let incoming = mask >> (<$ty>::BITS - 1) != 0;
                             let (expected, first_borrow) = lhs.overflowing_sub(rhs);
                             let (expected, second_borrow) =
                                 expected.overflowing_sub(<$ty>::from(incoming));
@@ -329,15 +371,35 @@ mod tests {
                                 (difference, borrow.to_bool()),
                                 (expected, first_borrow | second_borrow)
                             );
+                            assert_eq!(
+                                $sub_mask(lhs, rhs, mask),
+                                (
+                                    expected,
+                                    <$ty>::from(first_borrow | second_borrow).wrapping_neg()
+                                )
+                            );
                         }
                     }
                 }
             }
         };
     }
 
-    subtraction_tests!(sub_u32, u32, borrowing_sub_u32);
-    subtraction_tests!(sub_u64, u64, borrowing_sub_u64);
+    subtraction_tests!(sub_u32, u32, borrowing_sub_u32, borrowing_sub_mask_u32);
+    subtraction_tests!(sub_u64, u64, borrowing_sub_u64, borrowing_sub_mask_u64);
+
+    #[test]
+    fn sub_mask_propagates_borrow() {
+        let (lo, borrow) = borrowing_sub_mask_u32(0, 1, 0);
+        let (mid, borrow) = borrowing_sub_mask_u32(0, 0, borrow);
+        assert_eq!((lo, mid, borrow), (u32::MAX, u32::MAX, u32::MAX));
+        assert_eq!(borrowing_sub_mask_u32(1, 0, borrow), (0, 0));
+
+        let (lo, borrow) = borrowing_sub_mask_u64(0, 1, 0);
+        let (mid, borrow) = borrowing_sub_mask_u64(0, 0, borrow);
+        assert_eq!((lo, mid, borrow), (u64::MAX, u64::MAX, u64::MAX));
+        assert_eq!(borrowing_sub_mask_u64(1, 0, borrow), (0, 0));
+    }
 
     macro_rules! addition_tests {
         ($test:ident, $ty:ty, $add:ident) => {

ctutils/tests/primitives.rs

@@ -20,6 +20,46 @@ proptest! {
         prop_assert_eq!((difference, borrow.to_bool()), (expected, first_borrow | second_borrow));
     }
 
+    #[test]
+    fn sub_mask_u32(lhs in any::<u32>(), rhs in any::<u32>(), incoming in any::<u32>()) {
+        let (expected, first_borrow) = lhs.overflowing_sub(rhs);
+        let (expected, second_borrow) = expected.overflowing_sub(incoming >> 31);
+        let expected_borrow = u32::from(first_borrow | second_borrow).wrapping_neg();
+        prop_assert_eq!(borrowing_sub_mask_u32(lhs, rhs, incoming), (expected, expected_borrow));
+    }
+
+    #[test]
+    fn sub_mask_u64(lhs in any::<u64>(), rhs in any::<u64>(), incoming in any::<u64>()) {
+        let (expected, first_borrow) = lhs.overflowing_sub(rhs);
+        let (expected, second_borrow) = expected.overflowing_sub(incoming >> 63);
+        let expected_borrow = u64::from(first_borrow | second_borrow).wrapping_neg();
+        prop_assert_eq!(borrowing_sub_mask_u64(lhs, rhs, incoming), (expected, expected_borrow));
+    }
+
+    #[test]
+    #[allow(clippy::cast_possible_truncation, reason = "extract the low limb")]
+    fn sub_mask_u32_limbs(lhs in any::<u64>(), rhs in any::<u64>(), incoming in any::<u32>()) {
+        let (expected, first_borrow) = lhs.overflowing_sub(rhs);
+        let (expected, second_borrow) = expected.overflowing_sub(u64::from(incoming >> 31));
+        let expected_borrow = u32::from(first_borrow | second_borrow).wrapping_neg();
+        let (lo, borrow) = borrowing_sub_mask_u32(lhs as u32, rhs as u32, incoming);
+        let (hi, borrow) = borrowing_sub_mask_u32((lhs >> 32) as u32, (rhs >> 32) as u32, borrow);
+        let difference = (u64::from(hi) << 32) | u64::from(lo);
+        prop_assert_eq!((difference, borrow), (expected, expected_borrow));
+    }
+
+    #[test]
+    #[allow(clippy::cast_possible_truncation, reason = "extract the low limb")]
+    fn sub_mask_u64_limbs(lhs in any::<u128>(), rhs in any::<u128>(), incoming in any::<u64>()) {
+        let (expected, first_borrow) = lhs.overflowing_sub(rhs);
+        let (expected, second_borrow) = expected.overflowing_sub(u128::from(incoming >> 63));
+        let expected_borrow = u64::from(first_borrow | second_borrow).wrapping_neg();
+        let (lo, borrow) = borrowing_sub_mask_u64(lhs as u64, rhs as u64, incoming);
+        let (hi, borrow) = borrowing_sub_mask_u64((lhs >> 64) as u64, (rhs >> 64) as u64, borrow);
+        let difference = (u128::from(hi) << 64) | u128::from(lo);
+        prop_assert_eq!((difference, borrow), (expected, expected_borrow));
+    }
+
     #[test]
     fn zero_counts_u32(value in any::<u32>()) {
         prop_assert_eq!(leading_zeros_u32(value), value.leading_zeros());