From d6c029d7d53d582c33c5a60a42a793aee34012ee Mon Sep 17 00:00:00 2001 From: javis-bot Date: Sat, 13 Jun 2026 01:28:37 +0900 Subject: [PATCH] fix(bridge): keep decimals, versions and URLs whole in TTS sentence split MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The streaming splitter treated every "." as a sentence boundary, so the operational reply "17.5°C" was read as "17." / "5°C" and "1.8 km/h" as "1." / "8 km/h" - numbers spoken digit-by-digit plus extra TTS calls. An ASCII terminator (. ! ?) now only ends a sentence when it is followed by whitespace, a closing quote/bracket, or end of text. In-token dots (decimals "17.5", versions "v2.0", hosts "example.com") are followed by a digit/letter, so they no longer split. CJK fullwidth terminators stay unconditional since those scripts use no trailing space. Language-agnostic, punctuation only. - bridge: lookahead-gated boundary regex + finditer-based chunking - tests: regression cases for decimals (17.5/1.8), versions, URLs, and an integer that genuinely ends a sentence Co-Authored-By: Claude Opus 4.7 --- bridge/text_utils.py | 55 +++++++++++++++++++++-------- tests/test_bridge_sentence_split.py | 37 +++++++++++++++++++ 2 files changed, 77 insertions(+), 15 deletions(-) diff --git a/bridge/text_utils.py b/bridge/text_utils.py index 3440a3f..16fec45 100644 --- a/bridge/text_utils.py +++ b/bridge/text_utils.py @@ -9,38 +9,63 @@ from __future__ import annotations import re from typing import List, Optional -# Sentence terminators across scripts: ASCII . ! ? plus the CJK fullwidth forms -# and the ellipsis. Runs of terminators ("?!", "...") collapse into one boundary. -# A run of newlines is also a boundary. This is punctuation-only and therefore -# language-agnostic (no hardcoded words), per the project's multilingual rule. -_BOUNDARY = re.compile(r"([.!?。！？…]+|\n+)") +# A sentence boundary is one of: +# - a run of newlines, OR +# - a run of CJK fullwidth terminators (。！？) / the ellipsis (…) - these are +# ALWAYS boundaries because CJK scripts put no space after a sentence, OR +# - a run of ASCII terminators (. ! ?) that actually ENDS a sentence, i.e. is +# followed by whitespace, a closing quote/bracket, or the end of the text. +# +# Requiring that trailing whitespace/end for ASCII terminators is what keeps +# in-token dots from being mistaken for sentence ends, language-agnostically: +# - decimals -> "17.5°C", "1.8 km/h": the dot is followed by a digit, no space +# - versions -> "v2.0", "3.14": same +# - URLs/hosts-> "example.com/path": the dots are followed by letters, no space +# so none of them match and the number/URL stays inside a single spoken chunk. +# This is punctuation-only (no hardcoded words), per the project's multilingual +# rule. Runs of terminators ("?!", "...") still collapse into one boundary. +_BOUNDARY = re.compile( + r""" + (?P\n+) # a run of newlines + | (?P[。！？…]+) # CJK terminators: always end a sentence + | (?P[.!?]+) # ASCII terminator run... + (?=[)\]"'”’》」』]*(?:\s|$)) # ...only at a real sentence end + """, + re.VERBOSE, +) def split_sentences(text: Optional[str], min_len: int = 5) -> List[str]: """Split ``text`` into sentence-sized chunks for streaming TTS. Each chunk ends at a sentence boundary so it can be synthesised and played - while later chunks are still being spoken. Fragments shorter than - ``min_len`` characters (interjections like "네.", "응") are merged into an - adjacent chunk so we don't emit choppy micro-clips. Returns an empty list - for blank input and never loses visible content. + while later chunks are still being spoken. Sentence boundaries are detected + on terminal punctuation only (language-agnostic). Dots that live *inside* a + token - decimal points ("17.5"), version numbers ("v2.0") and URLs + ("example.com") - are NOT boundaries, so numbers and links are spoken in one + piece instead of being chopped digit-by-digit. + + Fragments shorter than ``min_len`` characters (interjections like "네.", and + single-letter initials like "J.") are merged into an adjacent chunk so we + don't emit choppy micro-clips. Returns an empty list for blank input and + never loses visible content. """ text = (text or "").strip() if not text: return [] - parts = _BOUNDARY.split(text) chunks: List[str] = [] buf = "" - for i in range(0, len(parts), 2): - seg = parts[i] - delim = parts[i + 1] if i + 1 < len(parts) else "" - buf += seg + delim + last = 0 + for m in _BOUNDARY.finditer(text): + buf += text[last:m.end()] + last = m.end() # Flush at a real boundary once the buffer is a worthwhile clip. - if delim and len(buf.strip()) >= min_len: + if len(buf.strip()) >= min_len: chunks.append(buf.strip()) buf = "" + buf += text[last:] tail = buf.strip() if tail: if chunks and len(tail) < min_len: diff --git a/tests/test_bridge_sentence_split.py b/tests/test_bridge_sentence_split.py index fe6ade7..1406506 100644 --- a/tests/test_bridge_sentence_split.py +++ b/tests/test_bridge_sentence_split.py @@ -75,3 +75,40 @@ def test_chunks_preserve_all_visible_content_in_order(): def test_collapses_repeated_terminators_into_one_chunk(): chunks = split_sentences("정말요?! 네 맞습니다.") assert chunks == ["정말요?!", "네 맞습니다."] + + +@pytest.mark.unit +def test_decimal_point_is_not_a_sentence_boundary(): + # Regression: "17.5" / "1.8" used to split as "17." / "5" and "1." / "8", + # making the TTS read numbers digit-by-digit. The decimal dot is followed by + # a digit (no space), so it must stay inside one chunk. + assert split_sentences("현재 기온은 17.5도입니다.") == ["현재 기온은 17.5도입니다."] + assert split_sentences("바람은 1.8 km/h입니다.") == ["바람은 1.8 km/h입니다."] + + +@pytest.mark.unit +def test_decimals_across_two_sentences_split_only_at_the_real_end(): + chunks = split_sentences("기온은 17.5도입니다. 바람은 1.8 km/h입니다.") + assert chunks == ["기온은 17.5도입니다.", "바람은 1.8 km/h입니다."] + + +@pytest.mark.unit +def test_english_decimal_and_version_numbers_stay_whole(): + assert split_sentences("Pi is about 3.14 today.") == ["Pi is about 3.14 today."] + assert split_sentences("Upgrade to v2.0 now.") == ["Upgrade to v2.0 now."] + + +@pytest.mark.unit +def test_url_dots_are_not_boundaries(): + # Dots inside a host/path are followed by letters, not whitespace, so the + # link is spoken in one piece; only the trailing sentence dot splits. + chunks = split_sentences("Visit example.com for details. Thanks!") + assert chunks == ["Visit example.com for details.", "Thanks!"] + + +@pytest.mark.unit +def test_integer_at_sentence_end_still_splits(): + # A dot after a number that genuinely ends a sentence (followed by a space) + # is still a boundary - only the *internal* decimal dot is protected. + chunks = split_sentences("I scored 5. Next round now.") + assert chunks == ["I scored 5.", "Next round now."]