fix(bridge): keep decimals, versions and URLs whole in TTS sentence split
The streaming splitter treated every "." as a sentence boundary, so the operational reply "17.5°C" was read as "17." / "5°C" and "1.8 km/h" as "1." / "8 km/h" - numbers spoken digit-by-digit plus extra TTS calls. An ASCII terminator (. ! ?) now only ends a sentence when it is followed by whitespace, a closing quote/bracket, or end of text. In-token dots (decimals "17.5", versions "v2.0", hosts "example.com") are followed by a digit/letter, so they no longer split. CJK fullwidth terminators stay unconditional since those scripts use no trailing space. Language-agnostic, punctuation only. - bridge: lookahead-gated boundary regex + finditer-based chunking - tests: regression cases for decimals (17.5/1.8), versions, URLs, and an integer that genuinely ends a sentence Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -9,38 +9,63 @@ from __future__ import annotations
|
||||
import re
|
||||
from typing import List, Optional
|
||||
|
||||
# Sentence terminators across scripts: ASCII . ! ? plus the CJK fullwidth forms
|
||||
# and the ellipsis. Runs of terminators ("?!", "...") collapse into one boundary.
|
||||
# A run of newlines is also a boundary. This is punctuation-only and therefore
|
||||
# language-agnostic (no hardcoded words), per the project's multilingual rule.
|
||||
_BOUNDARY = re.compile(r"([.!?。!?…]+|\n+)")
|
||||
# A sentence boundary is one of:
|
||||
# - a run of newlines, OR
|
||||
# - a run of CJK fullwidth terminators (。!?) / the ellipsis (…) - these are
|
||||
# ALWAYS boundaries because CJK scripts put no space after a sentence, OR
|
||||
# - a run of ASCII terminators (. ! ?) that actually ENDS a sentence, i.e. is
|
||||
# followed by whitespace, a closing quote/bracket, or the end of the text.
|
||||
#
|
||||
# Requiring that trailing whitespace/end for ASCII terminators is what keeps
|
||||
# in-token dots from being mistaken for sentence ends, language-agnostically:
|
||||
# - decimals -> "17.5°C", "1.8 km/h": the dot is followed by a digit, no space
|
||||
# - versions -> "v2.0", "3.14": same
|
||||
# - URLs/hosts-> "example.com/path": the dots are followed by letters, no space
|
||||
# so none of them match and the number/URL stays inside a single spoken chunk.
|
||||
# This is punctuation-only (no hardcoded words), per the project's multilingual
|
||||
# rule. Runs of terminators ("?!", "...") still collapse into one boundary.
|
||||
_BOUNDARY = re.compile(
|
||||
r"""
|
||||
(?P<nl>\n+) # a run of newlines
|
||||
| (?P<cjk>[。!?…]+) # CJK terminators: always end a sentence
|
||||
| (?P<ascii>[.!?]+) # ASCII terminator run...
|
||||
(?=[)\]"'”’》」』]*(?:\s|$)) # ...only at a real sentence end
|
||||
""",
|
||||
re.VERBOSE,
|
||||
)
|
||||
|
||||
|
||||
def split_sentences(text: Optional[str], min_len: int = 5) -> List[str]:
|
||||
"""Split ``text`` into sentence-sized chunks for streaming TTS.
|
||||
|
||||
Each chunk ends at a sentence boundary so it can be synthesised and played
|
||||
while later chunks are still being spoken. Fragments shorter than
|
||||
``min_len`` characters (interjections like "네.", "응") are merged into an
|
||||
adjacent chunk so we don't emit choppy micro-clips. Returns an empty list
|
||||
for blank input and never loses visible content.
|
||||
while later chunks are still being spoken. Sentence boundaries are detected
|
||||
on terminal punctuation only (language-agnostic). Dots that live *inside* a
|
||||
token - decimal points ("17.5"), version numbers ("v2.0") and URLs
|
||||
("example.com") - are NOT boundaries, so numbers and links are spoken in one
|
||||
piece instead of being chopped digit-by-digit.
|
||||
|
||||
Fragments shorter than ``min_len`` characters (interjections like "네.", and
|
||||
single-letter initials like "J.") are merged into an adjacent chunk so we
|
||||
don't emit choppy micro-clips. Returns an empty list for blank input and
|
||||
never loses visible content.
|
||||
"""
|
||||
text = (text or "").strip()
|
||||
if not text:
|
||||
return []
|
||||
|
||||
parts = _BOUNDARY.split(text)
|
||||
chunks: List[str] = []
|
||||
buf = ""
|
||||
for i in range(0, len(parts), 2):
|
||||
seg = parts[i]
|
||||
delim = parts[i + 1] if i + 1 < len(parts) else ""
|
||||
buf += seg + delim
|
||||
last = 0
|
||||
for m in _BOUNDARY.finditer(text):
|
||||
buf += text[last:m.end()]
|
||||
last = m.end()
|
||||
# Flush at a real boundary once the buffer is a worthwhile clip.
|
||||
if delim and len(buf.strip()) >= min_len:
|
||||
if len(buf.strip()) >= min_len:
|
||||
chunks.append(buf.strip())
|
||||
buf = ""
|
||||
|
||||
buf += text[last:]
|
||||
tail = buf.strip()
|
||||
if tail:
|
||||
if chunks and len(tail) < min_len:
|
||||
|
||||
Reference in New Issue
Block a user