From 682ec1825aa89d1b69233f0daffd50462d118aa9 Mon Sep 17 00:00:00 2001 From: Parker Timmerman Date: Sun, 12 Jul 2026 16:40:18 -0400 Subject: [PATCH 1/5] perf: copy small/medium strings inline instead of calling memcpy Add copy_small (inline path, <=24B overlapping-copy ladder) and copy_medium (heap path, [16,64] overlapping-SIMD bands, memcpy fallback otherwise). A runtime-length ptr::copy_nonoverlapping lowers to an out-of-line memcpy call; for these short copies the call boundary costs more than the copy itself. --- compact_str/src/repr/heap.rs | 10 ++--- compact_str/src/repr/inline.rs | 39 +------------------ compact_str/src/repr/mod.rs | 68 ++++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 45 deletions(-) diff --git a/compact_str/src/repr/heap.rs b/compact_str/src/repr/heap.rs index 22bc94802..845c53b52 100644 --- a/compact_str/src/repr/heap.rs +++ b/compact_str/src/repr/heap.rs @@ -43,7 +43,7 @@ impl HeapBuffer { // SAFETY: We know both `src` and `dest` are valid for respectively reads and writes of // length `len` because `len` comes from `src`, and `dest` was allocated to be at least that // length. We also know they're non-overlapping because `dest` is newly allocated - unsafe { ptr.as_ptr().copy_from_nonoverlapping(text.as_ptr(), len) }; + unsafe { super::copy_medium(text.as_ptr(), ptr.as_ptr(), len) }; Ok(HeapBuffer { ptr, len, cap }) } @@ -72,7 +72,7 @@ impl HeapBuffer { // SAFETY: We know both `src` and `dest` are valid for respectively reads and writes of // length `len` because `len` comes from `src`, and `dest` was allocated to be at least that // length. We also know they're non-overlapping because `dest` is newly allocated - unsafe { ptr.as_ptr().copy_from_nonoverlapping(text.as_ptr(), len) }; + unsafe { super::copy_medium(text.as_ptr(), ptr.as_ptr(), len) }; Ok(HeapBuffer { ptr, len, cap }) } @@ -237,11 +237,7 @@ impl Clone for HeapBuffer { // SAFETY: // * `src` and `dst` don't overlap because we just created `dst` // * `src` and `dst` are both valid for `self.len` bytes because self.len < capacity - unsafe { - new.ptr - .as_ptr() - .copy_from_nonoverlapping(self.ptr.as_ptr(), self.len) - }; + unsafe { super::copy_medium(self.ptr.as_ptr(), new.ptr.as_ptr(), self.len) }; // SAFETY: // * We copied the text from self, which is valid UTF-8 unsafe { new.set_len(self.len) }; diff --git a/compact_str/src/repr/inline.rs b/compact_str/src/repr/inline.rs index b62724156..16585bb7a 100644 --- a/compact_str/src/repr/inline.rs +++ b/compact_str/src/repr/inline.rs @@ -1,5 +1,3 @@ -use core::ptr; - use super::{Repr, LENGTH_MASK, MAX_SIZE}; /// A buffer stored on the stack whose size is equal to the stack size of `String` @@ -40,7 +38,7 @@ impl InlineBuffer { // * dst (`buffer`) is valid for `len` bytes because we assert src is less than MAX_SIZE // * src and dst don't overlap because we created dst // - copy_small(text.as_ptr(), buffer.0.as_mut_ptr(), len); + super::copy_small(text.as_ptr(), buffer.0.as_mut_ptr(), len); buffer } @@ -118,41 +116,6 @@ impl InlineBuffer { } } -/// Copies <= [`MAX_SIZE`] bytes `len` bytes from `src` to `dst`. -/// -/// `copy_nonoverlapping` with a runtime length emits a `memcpy` call into libc. This is -/// relatively expensive for the short copies used when inlining a string, which can -/// instead be implemented with a handfull of ASM instructions. -/// -/// Since an inline string is at most `MAX_SIZE` bytes, we use a ladder of constant-length -/// overlapping copies which LLVM can best optimize. -/// -/// # Safety -/// -/// * `len <= MAX_SIZE`. -/// * `src` and `dst` are valid for `len` bytes and do not overlap. -/// -#[inline(always)] -unsafe fn copy_small(src: *const u8, dst: *mut u8, len: usize) { - debug_assert!(len <= MAX_SIZE); - - if len >= 16 { - ptr::copy_nonoverlapping(src, dst, 16); - ptr::copy_nonoverlapping(src.add(len - 16), dst.add(len - 16), 16); - } else if len >= 8 { - ptr::copy_nonoverlapping(src, dst, 8); - ptr::copy_nonoverlapping(src.add(len - 8), dst.add(len - 8), 8); - } else if len >= 4 { - ptr::copy_nonoverlapping(src, dst, 4); - ptr::copy_nonoverlapping(src.add(len - 4), dst.add(len - 4), 4); - } else if len >= 2 { - ptr::copy_nonoverlapping(src, dst, 2); - ptr::copy_nonoverlapping(src.add(len - 2), dst.add(len - 2), 2); - } else if len == 1 { - *dst = *src; - } -} - #[cfg(test)] mod tests { #[test] diff --git a/compact_str/src/repr/mod.rs b/compact_str/src/repr/mod.rs index 3dd1e1d5d..ebba96c20 100644 --- a/compact_str/src/repr/mod.rs +++ b/compact_str/src/repr/mod.rs @@ -961,6 +961,74 @@ const unsafe fn assume(cond: bool) { } } +/// Copies `len` bytes (`len <= MAX_SIZE`) from `src` to `dst`, for the inline-string path. +/// +/// A `ptr::copy_nonoverlapping` with a *runtime* length lowers to an out-of-line `memcpy` call +/// (the backend only inlines the copy when the length is a compile-time constant), which is +/// wasteful for the tiny copies used when inlining a string. Since an inline string is at most +/// `MAX_SIZE` bytes we instead use a ladder of *constant*-length overlapping copies, each of which +/// the compiler lowers to a couple of load/store instructions — and which const-fold away entirely +/// when `len` is known (e.g. `CompactString::const_new` / string literals). +/// +/// # Safety +/// * `len <= MAX_SIZE`. +/// * `src` is valid for reads of `len` bytes; `dst` is valid for writes of `len` bytes. +/// * `src` and `dst` do not overlap. +#[inline(always)] +pub(crate) unsafe fn copy_small(src: *const u8, dst: *mut u8, len: usize) { + debug_assert!(len <= MAX_SIZE); + + // Each branch only touches bytes within `[0, len)` — the tail copy starts at `len - N` only + // once we know `len >= N` — so the overlapping copies never read or write out of range. + if len >= 16 { + ptr::copy_nonoverlapping(src, dst, 16); + ptr::copy_nonoverlapping(src.add(len - 16), dst.add(len - 16), 16); + } else if len >= 8 { + ptr::copy_nonoverlapping(src, dst, 8); + ptr::copy_nonoverlapping(src.add(len - 8), dst.add(len - 8), 8); + } else if len >= 4 { + ptr::copy_nonoverlapping(src, dst, 4); + ptr::copy_nonoverlapping(src.add(len - 4), dst.add(len - 4), 4); + } else if len >= 2 { + ptr::copy_nonoverlapping(src, dst, 2); + ptr::copy_nonoverlapping(src.add(len - 2), dst.add(len - 2), 2); + } else if len == 1 { + *dst = *src; + } +} + +/// Copies `len` bytes from `src` to `dst`, for the heap-string path. +/// +/// The common case for a freshly heap-allocated `CompactString` is a length just over +/// the inline threshold. For those the medium bands `[16,64]` are copied inline with two +/// overlapping fixed-size load/stores, skipping the `memcpy` call boundary like +/// [`copy_small`]. +/// +/// Copies `> 64` bytes delegate to `memcpy`, which has per-CPU-tuned large-copy paths +/// that any optimization we do won't beat. +/// Copies `< 16` bytes are rare on this path so they simply delegate to `memcpy` as well +/// which keeps this correct for *any* length. +/// +/// # Safety +/// +/// * `src` is valid for reads of `len` bytes; `dst` is valid for writes of `len` bytes. +/// * `src` and `dst` do not overlap. +/// +#[inline] +pub(crate) unsafe fn copy_medium(src: *const u8, dst: *mut u8, len: usize) { + if len > 64 { + ptr::copy_nonoverlapping(src, dst, len); + } else if len >= 32 { + ptr::copy_nonoverlapping(src, dst, 32); + ptr::copy_nonoverlapping(src.add(len - 32), dst.add(len - 32), 32); + } else if len >= 16 { + ptr::copy_nonoverlapping(src, dst, 16); + ptr::copy_nonoverlapping(src.add(len - 16), dst.add(len - 16), 16); + } else { + ptr::copy_nonoverlapping(src, dst, len); + } +} + #[cfg(test)] mod tests { use alloc::string::{String, ToString}; From 76477666e466167b96443baed10059637537dc01 Mon Sep 17 00:00:00 2001 From: Parker Timmerman Date: Sun, 12 Jul 2026 16:41:14 -0400 Subject: [PATCH 2/5] perf: keep HeapBuffer construction out-of-line Drop #[inline] from HeapBuffer::{new,with_capacity,with_additional}. These are the cold, allocating construction paths; keeping them out-of-line makes them a single shared function rather than being stamped into every CompactString construction callsite. --- compact_str/src/repr/heap.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/compact_str/src/repr/heap.rs b/compact_str/src/repr/heap.rs index 845c53b52..e1991bac7 100644 --- a/compact_str/src/repr/heap.rs +++ b/compact_str/src/repr/heap.rs @@ -32,8 +32,11 @@ static_assertions::assert_eq_size!(HeapBuffer, Repr); static_assertions::assert_eq_align!(HeapBuffer, Repr); impl HeapBuffer { - /// Create a [`HeapBuffer`] with the provided text - #[inline] + /// Create a [`HeapBuffer`] with the provided text. + /// + /// Note(parker): deliberately not `#[inline]`, this is the cold allocating heap + /// construction path, keeping it out-of-line prevents it from bloating every + /// `CompactString` construction callsite. pub(crate) fn new(text: &str) -> Result { let len = text.len(); let (cap, ptr) = allocate_ptr(len)?; @@ -49,7 +52,8 @@ impl HeapBuffer { } /// Create an empty [`HeapBuffer`] with a specific capacity - #[inline] + /// + /// Note: deliberately not `#[inline]` — see `HeapBuffer::new`. This is a cold allocating path. pub(crate) fn with_capacity(capacity: usize) -> Result { let len = 0; let (cap, ptr) = allocate_ptr(capacity)?; @@ -61,7 +65,8 @@ impl HeapBuffer { /// /// To prevent frequent re-allocations, this method will create a [`HeapBuffer`] with a capacity /// of `text.len() + additional` or `text.len() * 1.5`, whichever is greater - #[inline] + /// + /// Note: deliberately not `#[inline]` — see `HeapBuffer::new`. This is a cold allocating path. pub(crate) fn with_additional(text: &str, additional: usize) -> Result { let len = text.len(); let new_capacity = amortized_growth(len, additional); From a686f8ef7e98a9210213fec3ee3c01cf127d3924 Mon Sep 17 00:00:00 2001 From: Parker Timmerman Date: Sun, 12 Jul 2026 16:42:13 -0400 Subject: [PATCH 3/5] perf: move the OOM panic off the hot construction path Add HeapBuffer::new_panic (panics on allocation failure) and route Repr::new_panic through it. Since HeapBuffer construction is already out-of-line, the fallible-result check and the cold panic path now live in that shared function instead of being inlined into every callsite. A shared #[inline] build core keeps new/new_panic DRY without either calling the other. --- compact_str/src/repr/heap.rs | 26 ++++++++++++++++++++------ compact_str/src/repr/mod.rs | 3 ++- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/compact_str/src/repr/heap.rs b/compact_str/src/repr/heap.rs index e1991bac7..53db0e811 100644 --- a/compact_str/src/repr/heap.rs +++ b/compact_str/src/repr/heap.rs @@ -32,12 +32,10 @@ static_assertions::assert_eq_size!(HeapBuffer, Repr); static_assertions::assert_eq_align!(HeapBuffer, Repr); impl HeapBuffer { - /// Create a [`HeapBuffer`] with the provided text. - /// - /// Note(parker): deliberately not `#[inline]`, this is the cold allocating heap - /// construction path, keeping it out-of-line prevents it from bloating every - /// `CompactString` construction callsite. - pub(crate) fn new(text: &str) -> Result { + /// Allocate a fresh heap buffer and copy `text` into it, the shared core for + /// `new`/`new_panic`. + #[inline] + fn build(text: &str) -> Result { let len = text.len(); let (cap, ptr) = allocate_ptr(len)?; @@ -51,6 +49,22 @@ impl HeapBuffer { Ok(HeapBuffer { ptr, len, cap }) } + /// Create a [`HeapBuffer`] with the provided text. + /// + /// Note(parker): deliberately not `#[inline]`, this is the cold allocating heap + /// construction path, keeping it out-of-line prevents it from bloating every + /// `CompactString` construction callsite. + pub(crate) fn new(text: &str) -> Result { + Self::build(text) + } + + /// Like [`HeapBuffer::new`], but panics on allocation failure instead of returning a + /// `Result`. + #[track_caller] + pub(crate) fn new_panic(text: &str) -> Self { + Self::build(text).unwrap_with_msg() + } + /// Create an empty [`HeapBuffer`] with a specific capacity /// /// Note: deliberately not `#[inline]` — see `HeapBuffer::new`. This is a cold allocating path. diff --git a/compact_str/src/repr/mod.rs b/compact_str/src/repr/mod.rs index ebba96c20..20355ac89 100644 --- a/compact_str/src/repr/mod.rs +++ b/compact_str/src/repr/mod.rs @@ -95,7 +95,8 @@ impl Repr { let inline = unsafe { InlineBuffer::new(text) }; Repr::from_inline(inline) } else { - Repr::from_heap(HeapBuffer::new(text).unwrap_with_msg()) + let heap = HeapBuffer::new_panic(text); + Repr::from_heap(heap) } } From fac472ae951b8a23a975bb4cf415704521d428db Mon Sep 17 00:00:00 2001 From: Parker Timmerman Date: Sun, 12 Jul 2026 16:42:52 -0400 Subject: [PATCH 4/5] perf: drop the len == 0 fast path in Repr::new_panic An empty string flows through InlineBuffer::new to a byte-identical EMPTY repr (copy_small(_, _, 0) is a no-op), so the special case only added a branch to the hot construction path for no benefit. --- compact_str/src/repr/mod.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/compact_str/src/repr/mod.rs b/compact_str/src/repr/mod.rs index 20355ac89..77cf21e3a 100644 --- a/compact_str/src/repr/mod.rs +++ b/compact_str/src/repr/mod.rs @@ -88,9 +88,13 @@ impl Repr { pub(crate) fn new_panic(text: &str) -> Self { let len = text.len(); - if len == 0 { - EMPTY - } else if len <= MAX_SIZE { + // Note(parker): `len == 0` is intentionally not special-cased. + // + // I've gone back and forth on this a few times, but the slight performance win + // for empty strings is not worth the instruction bloat. `copy_small(_, _, 0)` in + // `InlineBuffer::new` ends up being a no-op as well. + + if len <= MAX_SIZE { // SAFETY: We checked that the length of text is less than or equal to MAX_SIZE let inline = unsafe { InlineBuffer::new(text) }; Repr::from_inline(inline) From 411948bfbc7d559df47634734502c18a31dc9c07 Mon Sep 17 00:00:00 2001 From: Parker Timmerman Date: Sun, 12 Jul 2026 16:42:53 -0400 Subject: [PATCH 5/5] bench: add 100 and 500 byte lengths to the creation benchmark Exercises the medium/large copy path (copy_medium and the memcpy fallback), not just the inline range. --- bench/benches/compact_str.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/benches/compact_str.rs b/bench/benches/compact_str.rs index 6fa3c9a1a..99d00ed0a 100644 --- a/bench/benches/compact_str.rs +++ b/bench/benches/compact_str.rs @@ -123,7 +123,7 @@ fn bench_str_to_compact_string(c: &mut Criterion) { fn bench_repr_creation(c: &mut Criterion) { let mut group = c.benchmark_group("Creation"); - let words: Vec = vec![0, 11, 12, 22, 23, 24, 25, 50] + let words: Vec = vec![0, 11, 12, 22, 23, 24, 25, 50, 100, 500] .into_iter() .map(|len| (0..len).into_iter().map(|_| 'a').collect()) .collect();