"""Small, dependency-free text helpers for the brain bridge. Kept separate from ``bridge.server`` (which imports Flask and the heavy brain) so the pure logic here can be unit-tested in isolation. """ from __future__ import annotations import re from typing import List, Optional # A sentence boundary is one of: # - a run of newlines, OR # - a run of CJK fullwidth terminators (。!?) / the ellipsis (…) - these are # ALWAYS boundaries because CJK scripts put no space after a sentence, OR # - a run of ASCII terminators (. ! ?) that actually ENDS a sentence, i.e. is # followed by whitespace, a closing quote/bracket, or the end of the text. # # Requiring that trailing whitespace/end for ASCII terminators is what keeps # in-token dots from being mistaken for sentence ends, language-agnostically: # - decimals -> "17.5°C", "1.8 km/h": the dot is followed by a digit, no space # - versions -> "v2.0", "3.14": same # - URLs/hosts-> "example.com/path": the dots are followed by letters, no space # so none of them match and the number/URL stays inside a single spoken chunk. # This is punctuation-only (no hardcoded words), per the project's multilingual # rule. Runs of terminators ("?!", "...") still collapse into one boundary. _BOUNDARY = re.compile( r""" (?P\n+) # a run of newlines | (?P[。!?…]+) # CJK terminators: always end a sentence | (?P[.!?]+) # ASCII terminator run... (?=[)\]"'”’》」』]*(?:\s|$)) # ...only at a real sentence end """, re.VERBOSE, ) def split_sentences(text: Optional[str], min_len: int = 5) -> List[str]: """Split ``text`` into sentence-sized chunks for streaming TTS. Each chunk ends at a sentence boundary so it can be synthesised and played while later chunks are still being spoken. Sentence boundaries are detected on terminal punctuation only (language-agnostic). Dots that live *inside* a token - decimal points ("17.5"), version numbers ("v2.0") and URLs ("example.com") - are NOT boundaries, so numbers and links are spoken in one piece instead of being chopped digit-by-digit. Fragments shorter than ``min_len`` characters (interjections like "네.", and single-letter initials like "J.") are merged into an adjacent chunk so we don't emit choppy micro-clips. Returns an empty list for blank input and never loses visible content. """ text = (text or "").strip() if not text: return [] chunks: List[str] = [] buf = "" last = 0 for m in _BOUNDARY.finditer(text): buf += text[last:m.end()] last = m.end() # Flush at a real boundary once the buffer is a worthwhile clip. if len(buf.strip()) >= min_len: chunks.append(buf.strip()) buf = "" buf += text[last:] tail = buf.strip() if tail: if chunks and len(tail) < min_len: chunks[-1] = chunks[-1] + " " + tail else: chunks.append(tail) return chunks