maybe_ascii: fold š/č/ş/ș/ț/ĉ and Cyrillic щ/ц to digraphs

rust/src/addresses/token.rs

@@ -86,8 +86,8 @@ mod tests {
     #[test]
     fn ascii_filled_for_cyrillic() {
         let tok = AddressToken::new("шоссе".to_string(), TokenClass::Text);
-        assert_eq!(tok.ascii.as_deref(), Some("sosse"));
-        assert_eq!(tok.comparable(), "sosse");
+        assert_eq!(tok.ascii.as_deref(), Some("shosse"));
+        assert_eq!(tok.comparable(), "shosse");
     }
 
     #[test]

rust/src/text/translit.rs

@@ -11,7 +11,7 @@
 
 use icu::experimental::transliterate::Transliterator;
 use icu::locale::Locale;
-use icu::normalizer::DecomposingNormalizerBorrowed;
+use icu::normalizer::{ComposingNormalizerBorrowed, DecomposingNormalizerBorrowed};
 use icu::properties::{CodePointMapData, props::GeneralCategory};
 use lru::LruCache;
 use std::cell::RefCell;
@@ -184,6 +184,51 @@ const ASCII_FALLBACK: &[(char, &str)] = &[
     ('ẟ', "d"), // small Latin delta (Latin Extended Additional)
 ];
 
+// Decomposable diacritics whose name spellings in the wild use a
+// digraph (`Kašerininov` ↔ `Kasherininov`, `Şükür` ↔ `Shukur`) rather
+// than the bare base letter. Must run before `nfkd_strip_marks`, which
+// would otherwise reduce them to `s`/`c`/`t`. Also catches ISO-9 output
+// of the Cyrillic pass (`ш → š`, `ч → č`). `ž` and `ţ` are deliberately
+// absent: measured on name-group spellings, `ž → zh` is a coin flip and
+// `ţ → t` beats `ţ → ts`.
+const DIGRAPH_FOLD: &[(char, &str)] = &[
+    ('š', "sh"),
+    ('Š', "Sh"),
+    ('č', "ch"),
+    ('Č', "Ch"),
+    ('ş', "sh"),
+    ('Ş', "Sh"),
+    ('ș', "sh"),
+    ('Ș', "Sh"),
+    ('ț', "ts"),
+    ('Ț', "Ts"),
+    ('ĉ', "ch"),
+    ('Ĉ', "Ch"),
+];
+
+// Cyrillic letters whose ISO-9 output loses the common ASCII spelling:
+// `щ → ŝ` would strip to `s`, and `ц → c` is already ASCII, so neither
+// can be fixed after the per-script pass. Substituted before it.
+const CYRILLIC_DIGRAPHS: &[(char, &str)] =
+    &[('щ', "shch"), ('Щ', "Shch"), ('ц', "ts"), ('Ц', "Ts")];
+
+fn replace_chars(input: &str, table: &[(char, &str)]) -> String {
+    let mut out = String::with_capacity(input.len() + 4);
+    for ch in input.chars() {
+        match table.iter().find(|(c, _)| *c == ch) {
+            Some((_, repl)) => out.push_str(repl),
+            None => out.push(ch),
+        }
+    }
+    out
+}
+
+fn digraph_fold(input: &str) -> String {
+    // NFC first so decomposed input (`s` + U+030C) hits the table.
+    let composed = ComposingNormalizerBorrowed::new_nfc().normalize(input);
+    replace_chars(&composed, DIGRAPH_FOLD)
+}
+
 fn ascii_fallback(input: &str) -> String {
     if input.is_ascii() {
         return input.to_string();
@@ -240,14 +285,17 @@ static MAYBE_ASCII_CACHE: LazyLock<Mutex<LruCache<String, Option<String>>>> =
 ///
 /// 1. **Per-script pass** (Cyrillic → Latin, Greek → Latin, etc.) —
 ///    non-Latin scripts go through their ICU4X transliterator to
-///    produce Latin output. Latin input skips this step.
-/// 2. **NFKD + nonspacing-mark strip** — decomposes base+combiner
+///    produce Latin output. Latin input skips this step. Cyrillic
+///    `щ → shch` and `ц → ts` are substituted before the ICU4X pass.
+/// 2. **Digraph fold** — `š`/`ş`/`ș → sh`, `č`/`ĉ → ch`, `ț → ts`,
+///    matching how these names are commonly spelled in ASCII.
+/// 3. **NFKD + nonspacing-mark strip** — decomposes base+combiner
 ///    sequences (é → e + acute → e) and resolves compatibility
 ///    variants (modifier letter ʱ → ɦ, superscript ᵋ → ɛ).
-/// 3. **CLDR Latin-ASCII pass** — applied unconditionally. Handles
+/// 4. **CLDR Latin-ASCII pass** — applied unconditionally. Handles
 ///    Latin Extended letters (`ĸ → q`, `ij → ij`, `ƙ → k`, `ɓ → b`,
-///    …) including the simplified base letters surfaced by step 2.
-/// 4. **`ASCII_FALLBACK`** — rigour's opinionated overrides for
+///    …) including the simplified base letters surfaced by step 3.
+/// 5. **`ASCII_FALLBACK`** — rigour's opinionated overrides for
 ///    cases where CLDR's Latin-ASCII keeps a non-ASCII letter
 ///    (e.g. Azerbaijani schwa `Ə → A`, African uppercase IPA letters
 ///    `Ʒ → Z`, `Ɔ → O`, …).
@@ -288,10 +336,14 @@ pub fn maybe_ascii(text: &str, drop: bool) -> String {
         if script == "Latin" {
             continue;
         }
+        if script == "Cyrillic" {
+            result = replace_chars(&result, CYRILLIC_DIGRAPHS);
+        }
         if let Some(locale_id) = locale_for_script(script) {
             result = transliterate_with(locale_id, result);
         }
     }
+    result = digraph_fold(&result);
     // NFKD first — compatibility decompositions turn modifier letters
     // and superscript Latin variants into their base letters (e.g.
     // ʱ → ɦ, ᵋ → ɛ). Running Latin-ASCII afterwards catches the
@@ -366,6 +418,26 @@ mod tests {
         assert_eq!(maybe_ascii("Zürich", false), "Zurich");
     }
 
+    #[test]
+    fn maybe_ascii_digraph_fold() {
+        assert_eq!(maybe_ascii("Kašerininov", false), "Kasherininov");
+        assert_eq!(maybe_ascii("Čopov", false), "Chopov");
+        assert_eq!(maybe_ascii("Şükür", false), "Shukur");
+        assert_eq!(maybe_ascii("Ștefan", false), "Shtefan");
+        assert_eq!(maybe_ascii("Țiriac", false), "Tsiriac");
+        assert_eq!(maybe_ascii("Ĉu", false), "Chu");
+        // Decomposed caron still folds to the digraph.
+        assert_eq!(maybe_ascii("s\u{030C}koda", false), "shkoda");
+        // ISO-9 output of the Cyrillic pass (ш → š, ч → č).
+        assert_eq!(maybe_ascii("Шевченко", false), "Shevchenko");
+        assert_eq!(maybe_ascii("Хрущёв", false), "Hrushchev");
+        assert_eq!(maybe_ascii("Цой", false), "Tsoj");
+        assert_eq!(maybe_ascii("Щукин", false), "Shchukin");
+        // Excluded: ž and ţ keep the bare-letter fold.
+        assert_eq!(maybe_ascii("Žukov", false), "Zukov");
+        assert_eq!(maybe_ascii("Ţara", false), "Tara");
+    }
+
     #[test]
     fn maybe_ascii_latin_fallback_table() {
         assert_eq!(maybe_ascii("Lars Løkke", false), "Lars Lokke");

tests/text/test_translit.py

@@ -70,6 +70,17 @@ def test_maybe_ascii_latin_diacritics():
     assert maybe_ascii("Zürich") == "Zurich"
 
 
+def test_maybe_ascii_digraph_fold():
+    assert maybe_ascii("Kašerininov") == "Kasherininov"
+    assert maybe_ascii("Čopov") == "Chopov"
+    assert maybe_ascii("Şükür") == "Shukur"
+    assert maybe_ascii("Țiriac") == "Tsiriac"
+    assert maybe_ascii("Шевченко") == "Shevchenko"
+    assert maybe_ascii("Хрущёв") == "Hrushchev"
+    assert maybe_ascii("Цой") == "Tsoj"
+    assert maybe_ascii("Žukov") == "Zukov"
+
+
 def test_maybe_ascii_latin_extended_fallback():
     # Cases that hit the fallback table / CLDR Latin-ASCII pass.
     assert maybe_ascii("weißbier") == "weissbier"