M6 + M0: hardening, deploy docs, PoC scripts

- Pipeline-down fallback: dialplan plays gogo-fallback after a failed AudioSocket (§15) + script to synthesize the Bosnian prompt via TTS - Voice heartbeat file → /health/voice endpoint + live status in admin panel - Retention job also purges stale call registrations; chat sessions metered; 100%-usage super-admin alert email (GOGO_ADMIN_ALERT_EMAIL) - .env.example, docs/DEPLOY.md (compose stack, GPU node, backups pg_dump, update procedure, security notes) - §16 Definition of Done as an automated test: softphone-style call → fake partner availability → request pushed → webhook confirm → console SMS - M0 PoC scripts: STT accuracy harness (CER/WER + spoken-digit check; dry-run verified with espeak-ng samples + faster-whisper small on CPU — phone number extracted exactly), TTS bake-off (Azure vs ElevenLabs, latency+cost), and end-to-end latency smoke test speaking real AudioSocket to the live pipeline Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-11 11:27:05 +02:00
parent de5a2abdeb
commit fd45ff84dc
16 changed files with 733 additions and 5 deletions
--- a/scripts/poc/latency_smoke.py
+++ b/scripts/poc/latency_smoke.py
@@ -0,0 +1,127 @@
+"""M0 PoC #3 — end-to-end latency smoke test (§14 Phase 0).
+
+Measures mouth-to-ear response time through the REAL pipeline: connects to the
+running AudioSocket voice server exactly like Asterisk would, plays a recorded
+utterance (or a tone with --fake providers), and measures the gap between
+end-of-utterance and the first agent audio frame of the reply.
+
+Target (§15): < 1.5 s, worst case < 2.5 s. GATE: > 2.5 s → rethink providers.
+
+Setup:
+    1. backend + voice server running (see docs/VOICE_TESTING.md)
+    2. a CallRegistration for the demo tenant is created automatically here
+
+Run:  python scripts/poc/latency_smoke.py [utterance.wav] [--host 127.0.0.1] [--port 9092]
+"""
+
+from __future__ import annotations
+
+import asyncio
+import sys
+import time
+import wave
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from sqlalchemy import select  # noqa: E402
+
+from gogo.db import get_sessionmaker  # noqa: E402
+from gogo.models import CallRegistration, Tenant  # noqa: E402
+from gogo.voice.audio import FRAME_BYTES, FRAME_MS, resample, silence, tone  # noqa: E402
+
+KIND_TERMINATE, KIND_UUID, KIND_AUDIO = 0x00, 0x01, 0x10
+
+
+def load_utterance(path: str | None) -> bytes:
+    if path:
+        with wave.open(path, "rb") as w:
+            pcm = w.readframes(w.getnframes())
+            if w.getframerate() != 8000:
+                pcm = resample(pcm, w.getframerate(), 8000)
+            return pcm
+    print("(no wav given — using a 1s tone; STT will hear nothing useful, "
+          "use a real recording for a true number)")
+    return tone(440, 1000)
+
+
+async def register_call() -> str:
+    async with get_sessionmaker()() as session:
+        tenant = (
+            await session.execute(select(Tenant).where(Tenant.slug == "salon-merima"))
+        ).scalar_one_or_none()
+        if tenant is None:
+            print("run `python -m gogo.seed` first")
+            sys.exit(1)
+        reg = CallRegistration(tenant_id=tenant.id, caller_msisdn="+38765123456")
+        session.add(reg)
+        await session.commit()
+        return str(reg.id)
+
+
+async def main() -> None:
+    args = [a for a in sys.argv[1:] if not a.startswith("--")]
+    host = sys.argv[sys.argv.index("--host") + 1] if "--host" in sys.argv else "127.0.0.1"
+    port = int(sys.argv[sys.argv.index("--port") + 1]) if "--port" in sys.argv else 9092
+
+    import uuid as uuid_mod
+
+    reg_id = uuid_mod.UUID(await register_call())
+    utterance = load_utterance(args[0] if args else None) + silence(800)
+
+    reader, writer = await asyncio.open_connection(host, port)
+    writer.write(bytes([KIND_UUID, 0, 16]) + reg_id.bytes)
+    await writer.drain()
+
+    async def read_msg():
+        header = await reader.readexactly(3)
+        length = int.from_bytes(header[1:3], "big")
+        payload = await reader.readexactly(length) if length else b""
+        return header[0], payload
+
+    # 1) greeting: time from handshake to first audio frame
+    t_start = time.monotonic()
+    kind, _ = await read_msg()
+    greeting_first_audio = time.monotonic() - t_start
+    print(f"greeting first audio after {greeting_first_audio * 1000:.0f} ms")
+
+    # drain the greeting while it plays (send silence like a real caller)
+    async def drain_audio(quiet_ms_needed: int = 600) -> None:
+        last_audio = time.monotonic()
+        while (time.monotonic() - last_audio) * 1000 < quiet_ms_needed:
+            try:
+                kind, _ = await asyncio.wait_for(read_msg(), timeout=0.05)
+                if kind == KIND_AUDIO:
+                    last_audio = time.monotonic()
+            except TimeoutError:
+                pass
+            writer.write(bytes([KIND_AUDIO]) + FRAME_BYTES.to_bytes(2, "big") + silence(FRAME_MS))
+            await writer.drain()
+            await asyncio.sleep(FRAME_MS / 1000)
+
+    await drain_audio()
+
+    # 2) speak the utterance paced like a phone, then measure to first reply frame
+    for i in range(0, len(utterance), FRAME_BYTES):
+        frame = utterance[i : i + FRAME_BYTES].ljust(FRAME_BYTES, b"\x00")
+        writer.write(bytes([KIND_AUDIO]) + len(frame).to_bytes(2, "big") + frame)
+        await writer.drain()
+        await asyncio.sleep(FRAME_MS / 1000)
+    t_end_of_speech = time.monotonic()
+
+    while True:
+        kind, payload = await read_msg()
+        if kind == KIND_AUDIO and payload.strip(b"\x00"):
+            break
+        if kind == KIND_TERMINATE:
+            print("server hung up before replying")
+            return
+    response_latency = time.monotonic() - t_end_of_speech
+    print(f"\nmouth-to-ear response latency: {response_latency * 1000:.0f} ms")
+    verdict = "OK ✓" if response_latency < 1.5 else ("worst-case ⚠" if response_latency < 2.5 else "FAIL ✗ — rethink providers (§14)")
+    print(f"target <1500 ms, worst case <2500 ms → {verdict}")
+    writer.close()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
--- a/scripts/poc/stt_test.py
+++ b/scripts/poc/stt_test.py
@@ -0,0 +1,112 @@
+"""M0 PoC #1 — STT accuracy on local speech samples (§14 Phase 0).
+
+Kill-the-risk test: can faster-whisper (language hint 'sr') reliably transcribe
+service names, personal names and PHONE NUMBERS SPOKEN ALOUD over phone-quality
+audio?
+
+Prepare a directory of WAV/MP3/OGG samples recorded by the founder + a
+manifest.csv with two columns:  filename,expected_text
+
+Run:
+    python scripts/poc/stt_test.py <samples_dir> [--model large-v3] [--device cuda]
+
+Outputs per-file transcript vs expected, character error rate (CER), word error
+rate (WER), digit accuracy (for phone numbers) and timing.
+GATE (§14): if phone numbers/names are unusable → stop and rethink providers.
+"""
+
+from __future__ import annotations
+
+import csv
+import re
+import sys
+import time
+from pathlib import Path
+
+
+def levenshtein(a: list, b: list) -> int:
+    prev = list(range(len(b) + 1))
+    for i, ca in enumerate(a, 1):
+        cur = [i]
+        for j, cb in enumerate(b, 1):
+            cur.append(min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (ca != cb)))
+        prev = cur
+    return prev[-1]
+
+
+def normalize(s: str) -> str:
+    return re.sub(r"[^\wšđčćž ]", "", s.lower()).strip()
+
+
+def digits(s: str) -> str:
+    words = {
+        "nula": "0", "jedan": "1", "dva": "2", "tri": "3", "četiri": "4",
+        "pet": "5", "šest": "6", "sedam": "7", "osam": "8", "devet": "9",
+    }
+    out = []
+    for token in normalize(s).split():
+        if token.isdigit():
+            out.append(token)
+        elif token in words:
+            out.append(words[token])
+    return "".join(out)
+
+
+def main() -> None:
+    if len(sys.argv) < 2:
+        print(__doc__)
+        sys.exit(1)
+    samples_dir = Path(sys.argv[1])
+    model_name = sys.argv[sys.argv.index("--model") + 1] if "--model" in sys.argv else "large-v3"
+    device = sys.argv[sys.argv.index("--device") + 1] if "--device" in sys.argv else "auto"
+
+    manifest = samples_dir / "manifest.csv"
+    if not manifest.exists():
+        print(f"missing {manifest} (columns: filename,expected_text)")
+        sys.exit(1)
+
+    from faster_whisper import WhisperModel
+
+    print(f"loading {model_name} on {device}…")
+    t0 = time.monotonic()
+    model = WhisperModel(model_name, device=device,
+                         compute_type="float16" if device == "cuda" else "int8")
+    print(f"model loaded in {time.monotonic() - t0:.1f}s\n")
+
+    rows = list(csv.DictReader(manifest.open()))
+    total_cer = total_wer = 0.0
+    digit_ok = digit_total = 0
+    for row in rows:
+        path = samples_dir / row["filename"]
+        expected = row["expected_text"]
+        t0 = time.monotonic()
+        segments, info = model.transcribe(str(path), language="sr", beam_size=5, vad_filter=True)
+        got = " ".join(s.text.strip() for s in segments).strip()
+        elapsed = time.monotonic() - t0
+
+        exp_n, got_n = normalize(expected), normalize(got)
+        cer = levenshtein(list(exp_n), list(got_n)) / max(1, len(exp_n))
+        wer = levenshtein(exp_n.split(), got_n.split()) / max(1, len(exp_n.split()))
+        total_cer += cer
+        total_wer += wer
+
+        exp_digits = digits(expected)
+        if exp_digits:
+            digit_total += 1
+            digit_ok += exp_digits == digits(got)
+
+        flag = "✓" if wer < 0.2 else ("~" if wer < 0.5 else "✗")
+        print(f"{flag} {row['filename']}  ({elapsed:.1f}s, audio {info.duration:.1f}s)")
+        print(f"   očekivano: {expected}")
+        print(f"   dobijeno : {got}\n")
+
+    n = max(1, len(rows))
+    print("=" * 60)
+    print(f"samples: {len(rows)}  avg CER: {total_cer / n:.1%}  avg WER: {total_wer / n:.1%}")
+    if digit_total:
+        print(f"phone-number samples exactly right: {digit_ok}/{digit_total}")
+    print("GATE: WER > 50% on names or wrong digits on most numbers → rethink STT (§14)")
+
+
+if __name__ == "__main__":
+    main()
--- a/scripts/poc/tts_bakeoff.py
+++ b/scripts/poc/tts_bakeoff.py
@@ -0,0 +1,82 @@
+"""M0 PoC #2 — TTS bake-off: Azure Neural vs ElevenLabs Flash (§14 Phase 0).
+
+Synthesizes typical agent sentences with every candidate voice, saves WAVs for
+the founder to listen to, and prints latency + rough per-character cost.
+
+Env: GOGO_AZURE_SPEECH_KEY / GOGO_AZURE_SPEECH_REGION and/or GOGO_ELEVENLABS_API_KEY.
+Run:  python scripts/poc/tts_bakeoff.py [out_dir]
+"""
+
+from __future__ import annotations
+
+import asyncio
+import sys
+import time
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from gogo.config import get_settings  # noqa: E402
+from gogo.voice.audio import pcm_to_wav  # noqa: E402
+from gogo.voice.tts import AzureTTS, ElevenLabsTTS  # noqa: E402
+
+SENTENCES = [
+    "Dobar dan, dobili ste salon Merima. Ja sam Gogo, virtuelni asistent — razgovor se snima. Kako vam mogu pomoći?",
+    "U srijedu poslijepodne slobodno je u dva i trideset ili u pet. Šta vam više odgovara?",
+    "Farbanje cijele dužine je od šezdeset do devedeset maraka, zavisno od dužine kose.",
+    "Važi. Na koje ime da zavedem zahtjev?",
+    "Prosljeđujem salonu zahtjev: šišanje i feniranje, srijeda u pet. Kontaktiraće vas u najkraćem roku radi potvrde. Hvala na pozivu i prijatno!",
+    "Izvinite, nisam vas dobro razumio. Možete li ponoviti?",
+]
+
+AZURE_VOICES = ["sr-RS-SophieNeural", "sr-RS-NicholasNeural", "hr-HR-GabrijelaNeural", "hr-HR-SreckoNeural"]
+ELEVEN_VOICES = ["JBFqnCBsd6RMkjVDRZzb"]  # replace with shortlisted hr/sr voices
+
+# rough public prices (2026): Azure Neural ~$15/1M chars, ElevenLabs Flash ~$50/1M chars
+COST_PER_CHAR_USD = {"azure": 15 / 1_000_000, "elevenlabs": 50 / 1_000_000}
+
+
+async def bake(provider_name: str, provider, voices: list[str], out: Path) -> None:
+    for voice in voices:
+        total_chars = total_ms = 0
+        for i, sentence in enumerate(SENTENCES, 1):
+            t0 = time.monotonic()
+            try:
+                pcm = await provider.synthesize(sentence, voice)
+            except Exception as e:  # noqa: BLE001
+                print(f"  {voice} #{i}: FAILED — {e}")
+                continue
+            ms = int((time.monotonic() - t0) * 1000)
+            total_chars += len(sentence)
+            total_ms += ms
+            path = out / f"{provider_name}-{voice.replace(':', '_')}-{i}.wav"
+            path.write_bytes(pcm_to_wav(pcm))
+            print(f"  {voice} #{i}: {ms} ms, {len(pcm) // 16} ms audio → {path.name}")
+        if total_chars:
+            cost = total_chars * COST_PER_CHAR_USD[provider_name]
+            print(
+                f"  {voice}: avg latency {total_ms // len(SENTENCES)} ms, "
+                f"~${cost:.4f} for all {total_chars} chars "
+                f"(~${cost / len(SENTENCES) * 1000:.3f} per 1000-char call)\n"
+            )
+
+
+async def main() -> None:
+    out = Path(sys.argv[1] if len(sys.argv) > 1 else "./poc-tts-out")
+    out.mkdir(parents=True, exist_ok=True)
+    s = get_settings()
+    if s.azure_speech_key:
+        print("== Azure Neural ==")
+        await bake("azure", AzureTTS(), AZURE_VOICES, out)
+    else:
+        print("(GOGO_AZURE_SPEECH_KEY not set — skipping Azure)")
+    if s.elevenlabs_api_key:
+        print("== ElevenLabs Flash ==")
+        await bake("elevenlabs", ElevenLabsTTS(), ELEVEN_VOICES, out)
+    else:
+        print("(GOGO_ELEVENLABS_API_KEY not set — skipping ElevenLabs)")
+    print(f"\nListen to the WAVs in {out}/ and pick 2–3 voices for the dashboard (§6.1).")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())