fix(bridge): keep decimals, versions and URLs whole in TTS sentence split

The streaming splitter treated every "." as a sentence boundary, so the
operational reply "17.5°C" was read as "17." / "5°C" and "1.8 km/h" as
"1." / "8 km/h" - numbers spoken digit-by-digit plus extra TTS calls.

An ASCII terminator (. ! ?) now only ends a sentence when it is followed by
whitespace, a closing quote/bracket, or end of text. In-token dots (decimals
"17.5", versions "v2.0", hosts "example.com") are followed by a digit/letter,
so they no longer split. CJK fullwidth terminators stay unconditional since
those scripts use no trailing space. Language-agnostic, punctuation only.

- bridge: lookahead-gated boundary regex + finditer-based chunking
- tests: regression cases for decimals (17.5/1.8), versions, URLs, and an
  integer that genuinely ends a sentence

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-06-13 01:28:37 +09:00
parent 5c295420ea
commit d6c029d7d5
2 changed files with 77 additions and 15 deletions

View File

@@ -9,38 +9,63 @@ from __future__ import annotations
import re import re
from typing import List, Optional from typing import List, Optional
# Sentence terminators across scripts: ASCII . ! ? plus the CJK fullwidth forms # A sentence boundary is one of:
# and the ellipsis. Runs of terminators ("?!", "...") collapse into one boundary. # - a run of newlines, OR
# A run of newlines is also a boundary. This is punctuation-only and therefore # - a run of CJK fullwidth terminators (。!?) / the ellipsis (…) - these are
# language-agnostic (no hardcoded words), per the project's multilingual rule. # ALWAYS boundaries because CJK scripts put no space after a sentence, OR
_BOUNDARY = re.compile(r"([.!?。!?…]+|\n+)") # - a run of ASCII terminators (. ! ?) that actually ENDS a sentence, i.e. is
# followed by whitespace, a closing quote/bracket, or the end of the text.
#
# Requiring that trailing whitespace/end for ASCII terminators is what keeps
# in-token dots from being mistaken for sentence ends, language-agnostically:
# - decimals -> "17.5°C", "1.8 km/h": the dot is followed by a digit, no space
# - versions -> "v2.0", "3.14": same
# - URLs/hosts-> "example.com/path": the dots are followed by letters, no space
# so none of them match and the number/URL stays inside a single spoken chunk.
# This is punctuation-only (no hardcoded words), per the project's multilingual
# rule. Runs of terminators ("?!", "...") still collapse into one boundary.
_BOUNDARY = re.compile(
r"""
(?P<nl>\n+) # a run of newlines
| (?P<cjk>[。!?…]+) # CJK terminators: always end a sentence
| (?P<ascii>[.!?]+) # ASCII terminator run...
(?=[)\]"'”’》」』]*(?:\s|$)) # ...only at a real sentence end
""",
re.VERBOSE,
)
def split_sentences(text: Optional[str], min_len: int = 5) -> List[str]: def split_sentences(text: Optional[str], min_len: int = 5) -> List[str]:
"""Split ``text`` into sentence-sized chunks for streaming TTS. """Split ``text`` into sentence-sized chunks for streaming TTS.
Each chunk ends at a sentence boundary so it can be synthesised and played Each chunk ends at a sentence boundary so it can be synthesised and played
while later chunks are still being spoken. Fragments shorter than while later chunks are still being spoken. Sentence boundaries are detected
``min_len`` characters (interjections like "네.", "") are merged into an on terminal punctuation only (language-agnostic). Dots that live *inside* a
adjacent chunk so we don't emit choppy micro-clips. Returns an empty list token - decimal points ("17.5"), version numbers ("v2.0") and URLs
for blank input and never loses visible content. ("example.com") - are NOT boundaries, so numbers and links are spoken in one
piece instead of being chopped digit-by-digit.
Fragments shorter than ``min_len`` characters (interjections like "네.", and
single-letter initials like "J.") are merged into an adjacent chunk so we
don't emit choppy micro-clips. Returns an empty list for blank input and
never loses visible content.
""" """
text = (text or "").strip() text = (text or "").strip()
if not text: if not text:
return [] return []
parts = _BOUNDARY.split(text)
chunks: List[str] = [] chunks: List[str] = []
buf = "" buf = ""
for i in range(0, len(parts), 2): last = 0
seg = parts[i] for m in _BOUNDARY.finditer(text):
delim = parts[i + 1] if i + 1 < len(parts) else "" buf += text[last:m.end()]
buf += seg + delim last = m.end()
# Flush at a real boundary once the buffer is a worthwhile clip. # Flush at a real boundary once the buffer is a worthwhile clip.
if delim and len(buf.strip()) >= min_len: if len(buf.strip()) >= min_len:
chunks.append(buf.strip()) chunks.append(buf.strip())
buf = "" buf = ""
buf += text[last:]
tail = buf.strip() tail = buf.strip()
if tail: if tail:
if chunks and len(tail) < min_len: if chunks and len(tail) < min_len:

View File

@@ -75,3 +75,40 @@ def test_chunks_preserve_all_visible_content_in_order():
def test_collapses_repeated_terminators_into_one_chunk(): def test_collapses_repeated_terminators_into_one_chunk():
chunks = split_sentences("정말요?! 네 맞습니다.") chunks = split_sentences("정말요?! 네 맞습니다.")
assert chunks == ["정말요?!", "네 맞습니다."] assert chunks == ["정말요?!", "네 맞습니다."]
@pytest.mark.unit
def test_decimal_point_is_not_a_sentence_boundary():
# Regression: "17.5" / "1.8" used to split as "17." / "5" and "1." / "8",
# making the TTS read numbers digit-by-digit. The decimal dot is followed by
# a digit (no space), so it must stay inside one chunk.
assert split_sentences("현재 기온은 17.5도입니다.") == ["현재 기온은 17.5도입니다."]
assert split_sentences("바람은 1.8 km/h입니다.") == ["바람은 1.8 km/h입니다."]
@pytest.mark.unit
def test_decimals_across_two_sentences_split_only_at_the_real_end():
chunks = split_sentences("기온은 17.5도입니다. 바람은 1.8 km/h입니다.")
assert chunks == ["기온은 17.5도입니다.", "바람은 1.8 km/h입니다."]
@pytest.mark.unit
def test_english_decimal_and_version_numbers_stay_whole():
assert split_sentences("Pi is about 3.14 today.") == ["Pi is about 3.14 today."]
assert split_sentences("Upgrade to v2.0 now.") == ["Upgrade to v2.0 now."]
@pytest.mark.unit
def test_url_dots_are_not_boundaries():
# Dots inside a host/path are followed by letters, not whitespace, so the
# link is spoken in one piece; only the trailing sentence dot splits.
chunks = split_sentences("Visit example.com for details. Thanks!")
assert chunks == ["Visit example.com for details.", "Thanks!"]
@pytest.mark.unit
def test_integer_at_sentence_end_still_splits():
# A dot after a number that genuinely ends a sentence (followed by a space)
# is still a boundary - only the *internal* decimal dot is protected.
chunks = split_sentences("I scored 5. Next round now.")
assert chunks == ["I scored 5.", "Next round now."]