Share UTF-8 sources across styled text layouts

CHANGES.md

@@ -1,5 +1,21 @@
 # Changes
 
+## 0.42.0
+
+- Add immutable shared glyph arrangements and glyph-range views for line render
+  nodes. Frozen arrangements discard redundant position and selection arrays
+  when arranged glyphs contain the same geometry.
+- Add `typesetSourceSpans` and its measurement variant for compact
+  `StyledTextRun` ranges and a separate `FontStyle` table. Layouts retain the
+  shared `Utf8Runes` source. Pixie reuses it for display text when unchanged
+  and maps converted or filtered glyphs back to original source byte ranges.
+  Harfbuzzy shapes normal-case ranges without keeping a string per styled run.
+- Expose sparse rune/byte position lookup on `Utf8Runes` so callers can create
+  byte ranges without a dense index.
+- Preserve the existing Pixie typesetting and arrangement-conversion proc
+  signatures, keep fonts and colors aligned when Pixie filters a whole control
+  span, and raise `ValueError` when borrowing bytes from a nil `Utf8Runes`.
+
 ## 0.41.0
 
 - Allocate Metal backdrop and rect-mask storage on first use, release optional

README.md

@@ -786,6 +786,37 @@ let layout = typeset(
 )
 ```
 
+For text that is already stored as one UTF-8 buffer, `typesetSourceSpans`
+accepts styled byte ranges and keeps that source in the returned layout:
+
+```nim
+let
+  source = initUtf8Runes("Hello שלום")
+  styles = [fs(bodyFont, fill(rgba(30, 34, 40, 255)))]
+  runs = [StyledTextRun(
+    byteStart: 0, byteEnd: source.byteLength.uint32,
+    runeStart: 0, runeEnd: source.len.uint32,
+    styleId: TextStyleId(0),
+  )]
+let layout = typesetSourceSpans(
+  rect(0, 0, 520, 120),
+  source,
+  runs,
+  styles,
+  minContent = false,
+  wrap = true,
+)
+```
+
+Ranges must cover the source in order, match their rune indexes, and refer to
+valid entries in the style table.
+`typesetSourceSpansForMeasurement` follows the same contract without publishing
+glyph images. Harfbuzzy shapes normal-case text with temporary strings for each
+shaping group; Pixie currently materializes strings for its span API. Both
+backends retain the original `Utf8Runes` as `sourceRunes`. Pixie also shares it
+as `runes` when display runes match the source. Case conversion and filtered
+controls require separate display runes, with glyph ranges mapped to source.
+
 Typeface metadata is parsed once when a face is registered and is available
 through the same API with every text backend:
 

figdraw.nimble

@@ -1,4 +1,4 @@
-version = "0.41.0"
+version = "0.42.0"
 author = "Jaremy Creechley"
 description = "UI Engine for Nim"
 license = "MIT"
@@ -7,6 +7,7 @@ srcDir = "src"
 # Dependencies
 requires "nim >= 2.2"
 requires "sigils >= 0.31.0"
+requires "threading >= 0.2.1"
 requires "pixie >= 5.0.1"
 requires "chroma >= 0.2.7"
 requires "bumpy"

src/figdraw/bindings/native_dynlib.json

@@ -134,6 +134,11 @@
     {"source": "../common/fonttypes.nim", "name": "len"},
     {"source": "../common/fonttypes.nim", "name": "isEmpty"},
     {"source": "../common/fonttypes.nim", "name": "stringValue"},
+    {"source": "../common/fonttypes.nim", "name": "byteLength"},
+    {"source": "../common/fonttypes.nim", "name": "runeCheckpointCount"},
+    {"source": "../common/fonttypes.nim", "name": "byteOffsetForRune"},
+    {"source": "../common/fonttypes.nim", "name": "runeIndexAtOrBeforeByte"},
+    {"source": "../common/fonttypes.nim", "name": "runeIndexAtOrAfterByte"},
     {"source": "../common/fonttypes.nim", "name": "utf8RunesFromText"},
     {"source": "../common/fonttypes.nim", "name": "initUtf8Runes"},
     {"source": "../common/fonttypes.nim", "name": "copyUtf8Runes"},
@@ -158,6 +163,8 @@
     {"source": "../common/fontutils.nim", "name": "placeStyledGlyphs"},
     {"source": "../common/fontutils.nim", "name": "typesetStyled"},
     {"source": "../common/fontutils.nim", "name": "typesetStyledForMeasurement"},
+    {"source": "../common/fontutils.nim", "name": "typesetSourceSpans"},
+    {"source": "../common/fontutils.nim", "name": "typesetSourceSpansForMeasurement"},
     {"source": "../common/fontglyphs.nim", "name": "hash"},
     {"source": "../common/fontglyphs.nim", "name": "glyphs"},
     {"source": "../common/fontglyphs.nim", "name": "generateGlyph"},

src/figdraw/common/fontglyphs.nim

@@ -177,11 +177,7 @@ proc buildArrangedGlyphs*(
 
 iterator glyphs*(arrangement: GlyphArrangement): GlyphPosition =
   var idx = 0
-  let arrangedGlyphCount =
-    if arrangement.arrangedGlyphs.len > 0:
-      arrangement.arrangedGlyphs.len
-    else:
-      arrangement.runes.len
+  let arrangedGlyphCount = arrangement.glyphCount()
 
   block:
     for i, span in arrangement.spans:
@@ -200,8 +196,8 @@ iterator glyphs*(arrangement: GlyphArrangement): GlyphPosition =
           fill(rgba(0, 0, 0, 255))
       while idx < arrangedGlyphCount and idx in span:
         let arranged =
-          if arrangement.arrangedGlyphs.len > 0:
-            arrangement.arrangedGlyphs[idx]
+          if arrangement.isGlyphView() or arrangement.arrangedGlyphs.len > 0:
+            arrangement.arrangedGlyph(idx)
           else:
             let rune = arrangement.runes[idx]
             ArrangedGlyph(
@@ -247,28 +243,96 @@ proc generateGlyphImages*(arrangement: GlyphArrangement, lcdFiltering = false) =
   for glyph in arrangement.glyphs():
     glyph.generateGlyph(lcdFiltering = lcdFiltering)
 
-proc convertArrangement*(
+proc matchesDisplayRunes(source: Utf8Runes, display: openArray[Rune]): bool =
+  if source.isNil or source.len != display.len:
+    return false
+  var index = 0
+  for rune in source:
+    if rune != display[index]:
+      return false
+    inc index
+  true
+
+proc pixieHasDisplayRune(text: string): bool =
+  ## Match Pixie's control filtering so style metadata uses the same spans.
+  var byteIndex = 0
+  var rune: Rune
+  while byteIndex < text.len:
+    fastRuneAt(text, byteIndex, rune, true)
+    if rune.uint32 >= 32 or rune == Rune(10):
+      return true
+
+proc mapPixieSourceRanges(
+    glyphs: var seq[ArrangedGlyph], source: Utf8Runes, runs: openArray[StyledTextRun]
+) =
+  ## Pixie keeps one display rune per source rune except filtered controls.
+  ## Its font-case conversion can change byte widths, so use source positions.
+  let bytes = source.bytes
+  var glyphIndex = 0
+  for run in runs:
+    var
+      byteIndex = int(run.byteStart)
+      runeIndex = int(run.runeStart)
+    while byteIndex < int(run.byteEnd):
+      let
+        rune = bytes.runeAt(byteIndex)
+        nextByte = min(byteIndex + max(1, bytes.runeLenAt(byteIndex)), int(run.byteEnd))
+      if rune.uint32 >= 32 or rune == Rune(10):
+        if glyphIndex >= glyphs.len:
+          raise newException(ValueError, "Pixie display runes differ from source runs")
+        glyphs[glyphIndex].source = GlyphSourceRange(
+          byteStart: byteIndex,
+          byteEnd: nextByte,
+          runeStart: runeIndex,
+          runeEnd: runeIndex + 1,
+        )
+        inc glyphIndex
+      byteIndex = nextByte
+      inc runeIndex
+  if glyphIndex != glyphs.len:
+    raise newException(ValueError, "Pixie display runes differ from source runs")
+
+proc convertArrangementImpl(
     arrangement: Arrangement,
     box: Rect,
     uiSpans: openArray[(FontStyle, string)],
     hAlign: FontHorizontal,
     vAlign: FontVertical,
     gfonts: seq[GlyphFont],
-    minContent = false,
-    wrap = false,
+    minContent, wrap: bool,
+    sourceRunes: Utf8Runes,
+    sourceRuns: openArray[StyledTextRun],
 ): GlyphArrangement =
   var
     lines = newSeqOfCap[Slice[int]](arrangement.lines.len())
     spanSlices = newSeqOfCap[Slice[int]](arrangement.spans.len())
     selectionRects = newSeqOfCap[Rect](arrangement.selectionRects.len())
+    visibleFonts = newSeqOfCap[GlyphFont](arrangement.spans.len())
+    visibleColors = newSeqOfCap[Fill](arrangement.spans.len())
   for line in arrangement.lines:
     lines.add line[0] .. line[1]
   for span in arrangement.spans:
     spanSlices.add span[0] .. span[1]
   for rect in arrangement.selectionRects:
     selectionRects.add rect
+  for index, (style, text) in uiSpans:
+    if index < gfonts.len and text.pixieHasDisplayRune():
+      visibleFonts.add gfonts[index]
+      visibleColors.add style.color
+  if visibleFonts.len != arrangement.spans.len:
+    visibleFonts = gfonts
+    visibleColors = uiSpans.mapIt(it[0].color)
 
-  let storedRunes = initArrangementRunes(arrangement.runes)
+  let displayRunes =
+    if sourceRunes.matchesDisplayRunes(arrangement.runes):
+      sourceRunes
+    else:
+      initArrangementRunes(arrangement.runes)
+  var arrangedGlyphs = buildArrangedGlyphs(
+    displayRunes, arrangement.positions, selectionRects, spanSlices, visibleFonts
+  )
+  if not sourceRunes.isNil:
+    arrangedGlyphs.mapPixieSourceRanges(sourceRunes, sourceRuns)
 
   result = GlyphArrangement(
     contentHash: block:
@@ -278,13 +342,41 @@ proc convertArrangement*(
       !$h,
     lines: lines,
     spans: spanSlices,
-    fonts: gfonts,
-    spanColors: uiSpans.mapIt(it[0].color),
-    sourceRunes: storedRunes,
-    arrangedGlyphs: buildArrangedGlyphs(
-      storedRunes, arrangement.positions, selectionRects, spanSlices, gfonts
-    ),
-    runes: storedRunes,
+    fonts: visibleFonts,
+    spanColors: visibleColors,
+    sourceRunes: (if sourceRunes.isNil: displayRunes else: sourceRunes),
+    arrangedGlyphs: arrangedGlyphs,
+    runes: displayRunes,
     positions: arrangement.positions,
     selectionRects: selectionRects,
   )
+
+proc convertArrangement*(
+    arrangement: Arrangement,
+    box: Rect,
+    uiSpans: openArray[(FontStyle, string)],
+    hAlign: FontHorizontal,
+    vAlign: FontVertical,
+    gfonts: seq[GlyphFont],
+    minContent = false,
+    wrap = false,
+): GlyphArrangement =
+  convertArrangementImpl(
+    arrangement, box, uiSpans, hAlign, vAlign, gfonts, minContent, wrap, nil, []
+  )
+
+proc convertArrangementWithSource*(
+    arrangement: Arrangement,
+    box: Rect,
+    uiSpans: openArray[(FontStyle, string)],
+    hAlign: FontHorizontal,
+    vAlign: FontVertical,
+    gfonts: seq[GlyphFont],
+    minContent, wrap: bool,
+    sourceRunes: Utf8Runes,
+    sourceRuns: openArray[StyledTextRun],
+): GlyphArrangement =
+  convertArrangementImpl(
+    arrangement, box, uiSpans, hAlign, vAlign, gfonts, minContent, wrap, sourceRunes,
+    sourceRuns,
+  )