d6c029d7d5
The streaming splitter treated every "." as a sentence boundary, so the operational reply "17.5°C" was read as "17." / "5°C" and "1.8 km/h" as "1." / "8 km/h" - numbers spoken digit-by-digit plus extra TTS calls. An ASCII terminator (. ! ?) now only ends a sentence when it is followed by whitespace, a closing quote/bracket, or end of text. In-token dots (decimals "17.5", versions "v2.0", hosts "example.com") are followed by a digit/letter, so they no longer split. CJK fullwidth terminators stay unconditional since those scripts use no trailing space. Language-agnostic, punctuation only. - bridge: lookahead-gated boundary regex + finditer-based chunking - tests: regression cases for decimals (17.5/1.8), versions, URLs, and an integer that genuinely ends a sentence Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
115 lines
4.4 KiB
Python
115 lines
4.4 KiB
Python
"""Unit tests for the bridge sentence splitter that drives streaming TTS.
|
|
|
|
The splitter is the only new logic on the bridge's streaming path: it chops a
|
|
reply into sentence-sized chunks so the first sentence can be synthesised and
|
|
played while the rest are still being spoken. It must be language-agnostic
|
|
(punctuation only, no hardcoded words) per the project rule.
|
|
"""
|
|
|
|
import pytest
|
|
|
|
from bridge.text_utils import split_sentences
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_empty_text_yields_no_chunks():
|
|
assert split_sentences("") == []
|
|
assert split_sentences(" ") == []
|
|
assert split_sentences(None) == [] # type: ignore[arg-type]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_text_without_terminal_punctuation_is_one_chunk():
|
|
assert split_sentences("오늘 날씨 맑음") == ["오늘 날씨 맑음"]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_splits_on_sentence_ending_punctuation():
|
|
chunks = split_sentences("안녕하세요. 반갑습니다!")
|
|
assert chunks == ["안녕하세요.", "반갑습니다!"]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_splits_on_fullwidth_cjk_punctuation():
|
|
chunks = split_sentences("これはペンです。あれは何ですか？")
|
|
assert chunks == ["これはペンです。", "あれは何ですか？"]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_splits_english_sentences():
|
|
chunks = split_sentences("Hello there. How are you? I am fine!")
|
|
assert chunks == ["Hello there.", "How are you?", "I am fine!"]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_short_leading_fragment_merges_forward():
|
|
# "네." is below the min length, so it should ride along with the next
|
|
# sentence rather than become its own micro-clip.
|
|
chunks = split_sentences("네. 지금 바로 처리하겠습니다.")
|
|
assert chunks == ["네. 지금 바로 처리하겠습니다."]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_short_trailing_fragment_merges_backward():
|
|
chunks = split_sentences("지금 바로 처리하겠습니다. 응")
|
|
assert chunks == ["지금 바로 처리하겠습니다. 응"]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_newline_is_a_boundary():
|
|
chunks = split_sentences("첫 번째 줄입니다\n두 번째 줄입니다")
|
|
assert chunks == ["첫 번째 줄입니다", "두 번째 줄입니다"]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_chunks_preserve_all_visible_content_in_order():
|
|
text = "안녕하세요. 오늘 일정 알려드릴게요. 회의가 세 개 있습니다!"
|
|
chunks = split_sentences(text)
|
|
assert len(chunks) >= 2
|
|
# No content lost: stripping spaces, the concatenation matches the source.
|
|
joined = "".join(chunks)
|
|
assert joined.replace(" ", "") == text.replace(" ", "")
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_collapses_repeated_terminators_into_one_chunk():
|
|
chunks = split_sentences("정말요?! 네 맞습니다.")
|
|
assert chunks == ["정말요?!", "네 맞습니다."]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_decimal_point_is_not_a_sentence_boundary():
|
|
# Regression: "17.5" / "1.8" used to split as "17." / "5" and "1." / "8",
|
|
# making the TTS read numbers digit-by-digit. The decimal dot is followed by
|
|
# a digit (no space), so it must stay inside one chunk.
|
|
assert split_sentences("현재 기온은 17.5도입니다.") == ["현재 기온은 17.5도입니다."]
|
|
assert split_sentences("바람은 1.8 km/h입니다.") == ["바람은 1.8 km/h입니다."]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_decimals_across_two_sentences_split_only_at_the_real_end():
|
|
chunks = split_sentences("기온은 17.5도입니다. 바람은 1.8 km/h입니다.")
|
|
assert chunks == ["기온은 17.5도입니다.", "바람은 1.8 km/h입니다."]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_english_decimal_and_version_numbers_stay_whole():
|
|
assert split_sentences("Pi is about 3.14 today.") == ["Pi is about 3.14 today."]
|
|
assert split_sentences("Upgrade to v2.0 now.") == ["Upgrade to v2.0 now."]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_url_dots_are_not_boundaries():
|
|
# Dots inside a host/path are followed by letters, not whitespace, so the
|
|
# link is spoken in one piece; only the trailing sentence dot splits.
|
|
chunks = split_sentences("Visit example.com for details. Thanks!")
|
|
assert chunks == ["Visit example.com for details.", "Thanks!"]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_integer_at_sentence_end_still_splits():
|
|
# A dot after a number that genuinely ends a sentence (followed by a space)
|
|
# is still a boundary - only the *internal* decimal dot is protected.
|
|
chunks = split_sentences("I scored 5. Next round now.")
|
|
assert chunks == ["I scored 5.", "Next round now."]
|