M6 + M0: hardening, deploy docs, PoC scripts
- Pipeline-down fallback: dialplan plays gogo-fallback after a failed AudioSocket (§15) + script to synthesize the Bosnian prompt via TTS - Voice heartbeat file → /health/voice endpoint + live status in admin panel - Retention job also purges stale call registrations; chat sessions metered; 100%-usage super-admin alert email (GOGO_ADMIN_ALERT_EMAIL) - .env.example, docs/DEPLOY.md (compose stack, GPU node, backups pg_dump, update procedure, security notes) - §16 Definition of Done as an automated test: softphone-style call → fake partner availability → request pushed → webhook confirm → console SMS - M0 PoC scripts: STT accuracy harness (CER/WER + spoken-digit check; dry-run verified with espeak-ng samples + faster-whisper small on CPU — phone number extracted exactly), TTS bake-off (Azure vs ElevenLabs, latency+cost), and end-to-end latency smoke test speaking real AudioSocket to the live pipeline Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
112
scripts/poc/stt_test.py
Normal file
112
scripts/poc/stt_test.py
Normal file
@@ -0,0 +1,112 @@
|
||||
"""M0 PoC #1 — STT accuracy on local speech samples (§14 Phase 0).
|
||||
|
||||
Kill-the-risk test: can faster-whisper (language hint 'sr') reliably transcribe
|
||||
service names, personal names and PHONE NUMBERS SPOKEN ALOUD over phone-quality
|
||||
audio?
|
||||
|
||||
Prepare a directory of WAV/MP3/OGG samples recorded by the founder + a
|
||||
manifest.csv with two columns: filename,expected_text
|
||||
|
||||
Run:
|
||||
python scripts/poc/stt_test.py <samples_dir> [--model large-v3] [--device cuda]
|
||||
|
||||
Outputs per-file transcript vs expected, character error rate (CER), word error
|
||||
rate (WER), digit accuracy (for phone numbers) and timing.
|
||||
GATE (§14): if phone numbers/names are unusable → stop and rethink providers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def levenshtein(a: list, b: list) -> int:
|
||||
prev = list(range(len(b) + 1))
|
||||
for i, ca in enumerate(a, 1):
|
||||
cur = [i]
|
||||
for j, cb in enumerate(b, 1):
|
||||
cur.append(min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (ca != cb)))
|
||||
prev = cur
|
||||
return prev[-1]
|
||||
|
||||
|
||||
def normalize(s: str) -> str:
|
||||
return re.sub(r"[^\wšđčćž ]", "", s.lower()).strip()
|
||||
|
||||
|
||||
def digits(s: str) -> str:
|
||||
words = {
|
||||
"nula": "0", "jedan": "1", "dva": "2", "tri": "3", "četiri": "4",
|
||||
"pet": "5", "šest": "6", "sedam": "7", "osam": "8", "devet": "9",
|
||||
}
|
||||
out = []
|
||||
for token in normalize(s).split():
|
||||
if token.isdigit():
|
||||
out.append(token)
|
||||
elif token in words:
|
||||
out.append(words[token])
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if len(sys.argv) < 2:
|
||||
print(__doc__)
|
||||
sys.exit(1)
|
||||
samples_dir = Path(sys.argv[1])
|
||||
model_name = sys.argv[sys.argv.index("--model") + 1] if "--model" in sys.argv else "large-v3"
|
||||
device = sys.argv[sys.argv.index("--device") + 1] if "--device" in sys.argv else "auto"
|
||||
|
||||
manifest = samples_dir / "manifest.csv"
|
||||
if not manifest.exists():
|
||||
print(f"missing {manifest} (columns: filename,expected_text)")
|
||||
sys.exit(1)
|
||||
|
||||
from faster_whisper import WhisperModel
|
||||
|
||||
print(f"loading {model_name} on {device}…")
|
||||
t0 = time.monotonic()
|
||||
model = WhisperModel(model_name, device=device,
|
||||
compute_type="float16" if device == "cuda" else "int8")
|
||||
print(f"model loaded in {time.monotonic() - t0:.1f}s\n")
|
||||
|
||||
rows = list(csv.DictReader(manifest.open()))
|
||||
total_cer = total_wer = 0.0
|
||||
digit_ok = digit_total = 0
|
||||
for row in rows:
|
||||
path = samples_dir / row["filename"]
|
||||
expected = row["expected_text"]
|
||||
t0 = time.monotonic()
|
||||
segments, info = model.transcribe(str(path), language="sr", beam_size=5, vad_filter=True)
|
||||
got = " ".join(s.text.strip() for s in segments).strip()
|
||||
elapsed = time.monotonic() - t0
|
||||
|
||||
exp_n, got_n = normalize(expected), normalize(got)
|
||||
cer = levenshtein(list(exp_n), list(got_n)) / max(1, len(exp_n))
|
||||
wer = levenshtein(exp_n.split(), got_n.split()) / max(1, len(exp_n.split()))
|
||||
total_cer += cer
|
||||
total_wer += wer
|
||||
|
||||
exp_digits = digits(expected)
|
||||
if exp_digits:
|
||||
digit_total += 1
|
||||
digit_ok += exp_digits == digits(got)
|
||||
|
||||
flag = "✓" if wer < 0.2 else ("~" if wer < 0.5 else "✗")
|
||||
print(f"{flag} {row['filename']} ({elapsed:.1f}s, audio {info.duration:.1f}s)")
|
||||
print(f" očekivano: {expected}")
|
||||
print(f" dobijeno : {got}\n")
|
||||
|
||||
n = max(1, len(rows))
|
||||
print("=" * 60)
|
||||
print(f"samples: {len(rows)} avg CER: {total_cer / n:.1%} avg WER: {total_wer / n:.1%}")
|
||||
if digit_total:
|
||||
print(f"phone-number samples exactly right: {digit_ok}/{digit_total}")
|
||||
print("GATE: WER > 50% on names or wrong digits on most numbers → rethink STT (§14)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user