M4: voice pipeline (software-only telephony)
- AudioSocket server (asyncio TCP, Asterisk wire protocol) bridging calls into the same M2 agent used by chat - Call session engine: greeting → energy-VAD utterance collection → STT → agent turn → TTS playback with barge-in (120ms of caller speech stops playback); inactivity + max-duration guards; unintelligible audio reaches the agent as '[nerazumljivo]' so the two-attempt rule stays in the prompt - STTProvider (faster-whisper, lang hint 'sr', GPU/CPU via env) and TTSProvider (Azure Neural raw-8k PCM / ElevenLabs Flash) + test fakes - Recording (mixed caller+agent WAV), transcripts, per-turn latency trace, metering via record_agent_call (§12) - Internal dialplan API: /internal/calls/register (ring targets computed from working hours + ring settings §5.2), /answered (human outcome, no minutes), /hangup (abandoned → missed-call SMS §5.5); shared-token auth - Asterisk config generator (pjsip.conf + extensions.conf from DB): worker endpoints, per-tenant test-caller endpoint, register→Dial→AudioSocket dialplan with h-extension reporting — validated against a real Asterisk 20 container (modules, dialplan, endpoints all load) - docker-compose 'voice' profile (voice server + Asterisk), Dockerfile.voice, docs/VOICE_TESTING.md softphone runbook - 16 new tests incl. full fake-call e2e and a real-TCP AudioSocket wire test Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
0
gogo/voice/__init__.py
Normal file
0
gogo/voice/__init__.py
Normal file
94
gogo/voice/audio.py
Normal file
94
gogo/voice/audio.py
Normal file
@@ -0,0 +1,94 @@
|
||||
"""Audio helpers for the voice pipeline.
|
||||
|
||||
Wire format everywhere: SLIN — signed 16-bit LE mono PCM @ 8 kHz (AudioSocket).
|
||||
Whisper wants 16 kHz; TTS providers are asked for 8 kHz where possible and
|
||||
resampled otherwise.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import audioop
|
||||
import io
|
||||
import math
|
||||
import wave
|
||||
|
||||
SAMPLE_RATE = 8000
|
||||
SAMPLE_WIDTH = 2 # bytes, 16-bit
|
||||
FRAME_MS = 20
|
||||
FRAME_BYTES = SAMPLE_RATE * SAMPLE_WIDTH * FRAME_MS // 1000 # 320
|
||||
|
||||
|
||||
def resample(pcm: bytes, from_rate: int, to_rate: int) -> bytes:
|
||||
if from_rate == to_rate or not pcm:
|
||||
return pcm
|
||||
out, _ = audioop.ratecv(pcm, SAMPLE_WIDTH, 1, from_rate, to_rate, None)
|
||||
return out
|
||||
|
||||
|
||||
def rms(pcm: bytes) -> int:
|
||||
return audioop.rms(pcm, SAMPLE_WIDTH) if pcm else 0
|
||||
|
||||
|
||||
def duration_s(pcm: bytes, rate: int = SAMPLE_RATE) -> float:
|
||||
return len(pcm) / (rate * SAMPLE_WIDTH)
|
||||
|
||||
|
||||
def silence(ms: int, rate: int = SAMPLE_RATE) -> bytes:
|
||||
return b"\x00" * (rate * SAMPLE_WIDTH * ms // 1000)
|
||||
|
||||
|
||||
def tone(freq: int, ms: int, rate: int = SAMPLE_RATE, amplitude: int = 12000) -> bytes:
|
||||
"""Test helper: sine tone (registers as speech for the energy VAD)."""
|
||||
n = rate * ms // 1000
|
||||
return b"".join(
|
||||
int(amplitude * math.sin(2 * math.pi * freq * i / rate)).to_bytes(
|
||||
2, "little", signed=True
|
||||
)
|
||||
for i in range(n)
|
||||
)
|
||||
|
||||
|
||||
def pcm_to_wav(pcm: bytes, rate: int = SAMPLE_RATE) -> bytes:
|
||||
buf = io.BytesIO()
|
||||
with wave.open(buf, "wb") as w:
|
||||
w.setnchannels(1)
|
||||
w.setsampwidth(SAMPLE_WIDTH)
|
||||
w.setframerate(rate)
|
||||
w.writeframes(pcm)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def mix(a: bytes, b: bytes) -> bytes:
|
||||
"""Mix two PCM streams (unequal lengths ok)."""
|
||||
if len(a) < len(b):
|
||||
a, b = b, a
|
||||
if not b:
|
||||
return a
|
||||
mixed = audioop.add(a[: len(b)], b, SAMPLE_WIDTH)
|
||||
return mixed + a[len(b):]
|
||||
|
||||
|
||||
class CallRecorder:
|
||||
"""Accumulates caller + agent audio on a shared timeline and writes a WAV."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._caller = bytearray()
|
||||
self._agent = bytearray()
|
||||
|
||||
def add_caller(self, pcm: bytes) -> None:
|
||||
self._caller.extend(pcm)
|
||||
# keep agent track in sync (silence while caller talks)
|
||||
if len(self._agent) < len(self._caller):
|
||||
self._agent.extend(b"\x00" * (len(self._caller) - len(self._agent)))
|
||||
|
||||
def add_agent(self, pcm: bytes) -> None:
|
||||
self._agent.extend(pcm)
|
||||
if len(self._caller) < len(self._agent):
|
||||
self._caller.extend(b"\x00" * (len(self._agent) - len(self._caller)))
|
||||
|
||||
def to_wav(self) -> bytes:
|
||||
return pcm_to_wav(mix(bytes(self._caller), bytes(self._agent)))
|
||||
|
||||
@property
|
||||
def seconds(self) -> float:
|
||||
return duration_s(bytes(self._caller))
|
||||
246
gogo/voice/call.py
Normal file
246
gogo/voice/call.py
Normal file
@@ -0,0 +1,246 @@
|
||||
"""One agent-handled phone call: audio loop, barge-in, recording, metering (M4).
|
||||
|
||||
The transport hands us paced 20 ms SLIN frames from Asterisk (AudioSocket) and
|
||||
accepts frames to play back. Flow per §6.2: greeting → collect utterance (VAD)
|
||||
→ STT → agent turn (M2 loop, same tools as chat) → TTS → play, with barge-in
|
||||
(caller speech stops playback). Unintelligible audio reaches the agent as
|
||||
'[nerazumljivo]' so the two-attempts rule (§6.4/A.6) lives in the prompt.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from gogo.agent.loop import AgentConversation
|
||||
from gogo.config import get_settings
|
||||
from gogo.db import get_sessionmaker
|
||||
from gogo.metering import record_agent_call
|
||||
from gogo.models import Call, CallRegistration, Tenant, utcnow
|
||||
from gogo.voice.audio import FRAME_BYTES, FRAME_MS, CallRecorder
|
||||
from gogo.voice.stt import get_stt
|
||||
from gogo.voice.tts import get_tts
|
||||
from gogo.voice.vad import UtteranceCollector
|
||||
|
||||
log = logging.getLogger("gogo.voice.call")
|
||||
|
||||
UNCLEAR_MARKER = "[nerazumljivo]"
|
||||
GOODBYE_TIMEOUT = (
|
||||
"Izgleda da veza ne radi. Salon će vidjeti vaš broj i javiti vam se. Prijatno!"
|
||||
)
|
||||
|
||||
BARGE_IN_FRAMES = 6 # 120 ms of speech during playback stops it
|
||||
|
||||
|
||||
@dataclass
|
||||
class CallLimits:
|
||||
inactivity_s: float = 30.0
|
||||
max_call_s: float = 600.0
|
||||
max_turns: int = 30
|
||||
|
||||
|
||||
class Transport:
|
||||
"""What CallSession needs from the wire (implemented by AudioSocket + fakes)."""
|
||||
|
||||
async def read_frame(self) -> bytes | None:
|
||||
"""Next inbound 20 ms frame, or None on hangup."""
|
||||
raise NotImplementedError
|
||||
|
||||
async def send_audio(self, pcm: bytes) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def hangup(self) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class CallSession:
|
||||
def __init__(
|
||||
self,
|
||||
transport: Transport,
|
||||
registration_id: uuid.UUID,
|
||||
*,
|
||||
limits: CallLimits | None = None,
|
||||
pace: float = FRAME_MS / 1000,
|
||||
):
|
||||
self.transport = transport
|
||||
self.registration_id = registration_id
|
||||
self.limits = limits or CallLimits()
|
||||
self.pace = pace
|
||||
self.recorder = CallRecorder()
|
||||
self.collector = UtteranceCollector()
|
||||
self._started = time.monotonic()
|
||||
|
||||
async def run(self) -> None:
|
||||
async with get_sessionmaker()() as session:
|
||||
reg = (
|
||||
await session.execute(
|
||||
select(CallRegistration).where(CallRegistration.id == self.registration_id)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if reg is None:
|
||||
log.error("no registration for call %s — hanging up", self.registration_id)
|
||||
await self.transport.hangup()
|
||||
return
|
||||
tenant = (
|
||||
await session.execute(select(Tenant).where(Tenant.id == reg.tenant_id))
|
||||
).scalar_one()
|
||||
reg.handled_by_agent = True
|
||||
|
||||
call = Call(
|
||||
tenant_id=tenant.id,
|
||||
caller_msisdn=reg.caller_msisdn,
|
||||
started_at=utcnow(),
|
||||
)
|
||||
session.add(call)
|
||||
await session.flush()
|
||||
|
||||
convo = AgentConversation(
|
||||
session,
|
||||
tenant,
|
||||
channel="voice",
|
||||
caller_phone=reg.caller_msisdn,
|
||||
call_id=call.id,
|
||||
)
|
||||
tts, voice = get_tts(tenant.tts_voice)
|
||||
stt = get_stt()
|
||||
|
||||
try:
|
||||
greeting = await convo.greeting()
|
||||
await session.commit()
|
||||
await self._speak(tts, voice, greeting)
|
||||
|
||||
for _turn in range(self.limits.max_turns):
|
||||
if time.monotonic() - self._started > self.limits.max_call_s:
|
||||
log.warning("max call duration hit (tenant %s)", tenant.slug)
|
||||
break
|
||||
utterance = await self._collect_utterance()
|
||||
if utterance is None: # hangup
|
||||
break
|
||||
if utterance == b"": # inactivity
|
||||
await self._speak(tts, voice, GOODBYE_TIMEOUT)
|
||||
break
|
||||
|
||||
t0 = time.monotonic()
|
||||
text = await stt.transcribe(utterance)
|
||||
stt_ms = int((time.monotonic() - t0) * 1000)
|
||||
|
||||
t0 = time.monotonic()
|
||||
reply = await convo.user_turn(text or UNCLEAR_MARKER)
|
||||
llm_ms = int((time.monotonic() - t0) * 1000)
|
||||
await session.commit()
|
||||
|
||||
t0 = time.monotonic()
|
||||
interrupted = await self._speak(tts, voice, reply)
|
||||
tts_ms = int((time.monotonic() - t0) * 1000)
|
||||
|
||||
trace = call.trace or {}
|
||||
turns = trace.setdefault("turns", [])
|
||||
turns.append(
|
||||
{"stt_ms": stt_ms, "llm_ms": llm_ms, "speak_ms": tts_ms,
|
||||
"interrupted": interrupted, "chars": len(reply)}
|
||||
)
|
||||
call.trace = dict(trace)
|
||||
|
||||
if self._agent_closed(convo):
|
||||
break
|
||||
except Exception: # noqa: BLE001 — a crash must still finalize the call
|
||||
log.exception("call session error (tenant %s)", tenant.slug)
|
||||
finally:
|
||||
await self._finalize(session, tenant, call, convo, reg)
|
||||
await self.transport.hangup()
|
||||
|
||||
# -- audio helpers ------------------------------------------------------
|
||||
|
||||
async def _speak(self, tts, voice: str, text: str) -> bool:
|
||||
"""Synthesize and play; returns True if the caller barged in."""
|
||||
if not text:
|
||||
return False
|
||||
try:
|
||||
pcm = await tts.synthesize(text, voice)
|
||||
except Exception: # noqa: BLE001
|
||||
log.exception("TTS failed — continuing without audio")
|
||||
return False
|
||||
|
||||
voiced_streak = 0
|
||||
for i in range(0, len(pcm), FRAME_BYTES):
|
||||
frame = pcm[i : i + FRAME_BYTES]
|
||||
await self.transport.send_audio(frame)
|
||||
self.recorder.add_agent(frame)
|
||||
if self.pace:
|
||||
await asyncio.sleep(self.pace)
|
||||
# barge-in: watch inbound audio while we play
|
||||
inbound = await self._poll_frame()
|
||||
if inbound is not None:
|
||||
self.recorder.add_caller(inbound)
|
||||
if self.collector.is_speech(inbound):
|
||||
voiced_streak += 1
|
||||
if voiced_streak >= BARGE_IN_FRAMES:
|
||||
self.collector.reset()
|
||||
self.collector.feed(inbound)
|
||||
return True
|
||||
else:
|
||||
voiced_streak = 0
|
||||
return False
|
||||
|
||||
async def _poll_frame(self) -> bytes | None:
|
||||
try:
|
||||
return await asyncio.wait_for(self.transport.read_frame(), timeout=0.001)
|
||||
except TimeoutError:
|
||||
return None
|
||||
|
||||
async def _collect_utterance(self) -> bytes | None:
|
||||
"""None = hangup, b'' = inactivity timeout, else utterance PCM."""
|
||||
deadline = time.monotonic() + self.limits.inactivity_s
|
||||
while True:
|
||||
timeout = deadline - time.monotonic()
|
||||
if timeout <= 0:
|
||||
return b""
|
||||
try:
|
||||
frame = await asyncio.wait_for(self.transport.read_frame(), timeout=timeout)
|
||||
except TimeoutError:
|
||||
return b""
|
||||
if frame is None:
|
||||
return None
|
||||
self.recorder.add_caller(frame)
|
||||
utterance = self.collector.feed(frame)
|
||||
if utterance is not None:
|
||||
return utterance
|
||||
|
||||
def _agent_closed(self, convo: AgentConversation) -> bool:
|
||||
"""The closing line ends the call from our side (short calls §2.5)."""
|
||||
last = next((t.text for t in reversed(convo.turns) if t.role == "assistant"), "")
|
||||
lowered = last.lower()
|
||||
return ("prijatno" in lowered or "doviđenja" in lowered) and bool(
|
||||
convo.executor.created_request_id or convo.executor.took_message
|
||||
)
|
||||
|
||||
async def _finalize(self, session, tenant, call, convo, reg) -> None:
|
||||
try:
|
||||
call.duration_s = int(time.monotonic() - self._started)
|
||||
call.agent_seconds_charged = call.duration_s
|
||||
call.outcome = convo.outcome
|
||||
call.transcript = convo.transcript
|
||||
|
||||
rec_dir = get_settings().recordings_dir
|
||||
os.makedirs(rec_dir, exist_ok=True)
|
||||
path = os.path.join(rec_dir, f"{call.id}.wav")
|
||||
with open(path, "wb") as f:
|
||||
f.write(self.recorder.to_wav())
|
||||
call.recording_path = path
|
||||
|
||||
reg.finalized = True
|
||||
await record_agent_call(session, tenant, call.agent_seconds_charged)
|
||||
await session.commit()
|
||||
log.info(
|
||||
"call %s finalized: %ss outcome=%s tenant=%s",
|
||||
call.id, call.duration_s, call.outcome, tenant.slug,
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
log.exception("failed to finalize call %s", call.id)
|
||||
await session.rollback()
|
||||
113
gogo/voice/server.py
Normal file
113
gogo/voice/server.py
Normal file
@@ -0,0 +1,113 @@
|
||||
"""AudioSocket server — Asterisk connects here per agent-handled call (M4).
|
||||
|
||||
AudioSocket wire protocol (Asterisk `AudioSocket()` app):
|
||||
1 byte kind | 2 bytes big-endian payload length | payload
|
||||
kind 0x00 = terminate, 0x01 = UUID (16 raw bytes), 0x10 = SLIN audio,
|
||||
0xff = error. Audio is 16-bit LE mono PCM @ 8 kHz, ~20 ms per message.
|
||||
|
||||
Run: python -m gogo.voice.server
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import uuid
|
||||
|
||||
from gogo.config import get_settings
|
||||
from gogo.voice.call import CallSession, Transport
|
||||
|
||||
log = logging.getLogger("gogo.voice.server")
|
||||
|
||||
KIND_TERMINATE = 0x00
|
||||
KIND_UUID = 0x01
|
||||
KIND_AUDIO = 0x10
|
||||
KIND_ERROR = 0xFF
|
||||
|
||||
|
||||
class AudioSocketTransport(Transport):
|
||||
def __init__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter):
|
||||
self.reader = reader
|
||||
self.writer = writer
|
||||
self._closed = False
|
||||
|
||||
async def read_message(self) -> tuple[int, bytes] | None:
|
||||
try:
|
||||
header = await self.reader.readexactly(3)
|
||||
except (asyncio.IncompleteReadError, ConnectionResetError):
|
||||
return None
|
||||
kind = header[0]
|
||||
length = int.from_bytes(header[1:3], "big")
|
||||
payload = b""
|
||||
if length:
|
||||
try:
|
||||
payload = await self.reader.readexactly(length)
|
||||
except (asyncio.IncompleteReadError, ConnectionResetError):
|
||||
return None
|
||||
return kind, payload
|
||||
|
||||
async def read_frame(self) -> bytes | None:
|
||||
"""Next audio frame; skips non-audio messages; None on hangup/close."""
|
||||
while True:
|
||||
msg = await self.read_message()
|
||||
if msg is None:
|
||||
return None
|
||||
kind, payload = msg
|
||||
if kind == KIND_AUDIO:
|
||||
return payload
|
||||
if kind == KIND_TERMINATE:
|
||||
return None
|
||||
if kind == KIND_ERROR:
|
||||
log.warning("audiosocket error frame: %s", payload[:64])
|
||||
|
||||
async def send_audio(self, pcm: bytes) -> None:
|
||||
if self._closed:
|
||||
return
|
||||
try:
|
||||
self.writer.write(bytes([KIND_AUDIO]) + len(pcm).to_bytes(2, "big") + pcm)
|
||||
await self.writer.drain()
|
||||
except (ConnectionResetError, BrokenPipeError):
|
||||
self._closed = True
|
||||
|
||||
async def hangup(self) -> None:
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
try:
|
||||
self.writer.write(bytes([KIND_TERMINATE, 0, 0]))
|
||||
await self.writer.drain()
|
||||
self.writer.close()
|
||||
except (ConnectionResetError, BrokenPipeError):
|
||||
pass
|
||||
|
||||
|
||||
async def handle_connection(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
|
||||
transport = AudioSocketTransport(reader, writer)
|
||||
msg = await transport.read_message()
|
||||
if msg is None or msg[0] != KIND_UUID or len(msg[1]) != 16:
|
||||
log.warning("connection without UUID handshake — dropping")
|
||||
await transport.hangup()
|
||||
return
|
||||
registration_id = uuid.UUID(bytes=msg[1])
|
||||
log.info("call connected: %s", registration_id)
|
||||
session = CallSession(transport, registration_id)
|
||||
await session.run()
|
||||
|
||||
|
||||
async def serve() -> None:
|
||||
s = get_settings()
|
||||
server = await asyncio.start_server(handle_connection, s.voice_host, s.voice_port)
|
||||
addrs = ", ".join(str(sock.getsockname()) for sock in server.sockets)
|
||||
log.info("AudioSocket server listening on %s", addrs)
|
||||
# health heartbeat file for the admin panel / monitoring (M6)
|
||||
async with server:
|
||||
await server.serve_forever()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
||||
asyncio.run(serve())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
81
gogo/voice/stt.py
Normal file
81
gogo/voice/stt.py
Normal file
@@ -0,0 +1,81 @@
|
||||
"""STTProvider interface — faster-whisper implementation + test fake (§4).
|
||||
|
||||
Language hint 'sr' covers spoken Bosnian/Serbian/Croatian (§6.1). The model
|
||||
runs on GPU in the founder's DC (large-v3); CPU with a small model for dev.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from typing import Protocol
|
||||
|
||||
from gogo.voice.audio import SAMPLE_RATE, resample
|
||||
|
||||
log = logging.getLogger("gogo.voice.stt")
|
||||
|
||||
|
||||
class STTProvider(Protocol):
|
||||
async def transcribe(self, pcm_8k: bytes) -> str:
|
||||
"""SLIN 8 kHz PCM → text (empty string when nothing intelligible)."""
|
||||
...
|
||||
|
||||
|
||||
class WhisperSTT:
|
||||
"""faster-whisper; model size/device via env (GOGO_WHISPER_MODEL, GOGO_WHISPER_DEVICE)."""
|
||||
|
||||
def __init__(self, model_name: str | None = None, device: str | None = None):
|
||||
from faster_whisper import WhisperModel # heavy import, voice extra
|
||||
|
||||
model_name = model_name or os.environ.get("GOGO_WHISPER_MODEL", "small")
|
||||
device = device or os.environ.get("GOGO_WHISPER_DEVICE", "auto")
|
||||
compute = "float16" if device == "cuda" else "int8"
|
||||
log.info("loading whisper model=%s device=%s", model_name, device)
|
||||
self._model = WhisperModel(model_name, device=device, compute_type=compute)
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def transcribe(self, pcm_8k: bytes) -> str:
|
||||
import numpy as np
|
||||
|
||||
pcm_16k = resample(pcm_8k, SAMPLE_RATE, 16000)
|
||||
audio = np.frombuffer(pcm_16k, dtype=np.int16).astype(np.float32) / 32768.0
|
||||
|
||||
def run() -> str:
|
||||
segments, _info = self._model.transcribe(
|
||||
audio,
|
||||
language="sr", # covers bs/sr/hr as one spoken language (§6.1)
|
||||
beam_size=5,
|
||||
vad_filter=True,
|
||||
)
|
||||
return " ".join(s.text.strip() for s in segments).strip()
|
||||
|
||||
async with self._lock: # one decode at a time per worker process
|
||||
return await asyncio.get_running_loop().run_in_executor(None, run)
|
||||
|
||||
|
||||
class FakeSTT:
|
||||
"""Test fake: returns queued texts, one per transcribe() call."""
|
||||
|
||||
def __init__(self, texts: list[str] | None = None):
|
||||
self.texts = list(texts or [])
|
||||
self.received: list[bytes] = []
|
||||
|
||||
async def transcribe(self, pcm_8k: bytes) -> str:
|
||||
self.received.append(pcm_8k)
|
||||
return self.texts.pop(0) if self.texts else ""
|
||||
|
||||
|
||||
_stt: STTProvider | None = None
|
||||
|
||||
|
||||
def get_stt() -> STTProvider:
|
||||
global _stt
|
||||
if _stt is None:
|
||||
_stt = WhisperSTT()
|
||||
return _stt
|
||||
|
||||
|
||||
def set_stt(provider: STTProvider | None) -> None:
|
||||
global _stt
|
||||
_stt = provider
|
||||
107
gogo/voice/tts.py
Normal file
107
gogo/voice/tts.py
Normal file
@@ -0,0 +1,107 @@
|
||||
"""TTSProvider interface (§4): Azure Neural + ElevenLabs Flash + test fake.
|
||||
|
||||
Both return SLIN 8 kHz PCM ready for AudioSocket. Default provider/voice comes
|
||||
from settings; per-tenant voice ids look like "azure:sr-RS-SophieNeural" or
|
||||
"elevenlabs:<voice_id>".
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Protocol
|
||||
from xml.sax.saxutils import escape
|
||||
|
||||
import httpx
|
||||
|
||||
from gogo.config import get_settings
|
||||
from gogo.voice.audio import SAMPLE_RATE, resample, tone
|
||||
|
||||
log = logging.getLogger("gogo.voice.tts")
|
||||
|
||||
DEFAULT_AZURE_VOICE = "sr-RS-SophieNeural"
|
||||
|
||||
|
||||
class TTSProvider(Protocol):
|
||||
async def synthesize(self, text: str, voice: str = "") -> bytes:
|
||||
"""Text → SLIN 8 kHz PCM."""
|
||||
...
|
||||
|
||||
|
||||
class AzureTTS:
|
||||
async def synthesize(self, text: str, voice: str = "") -> bytes:
|
||||
s = get_settings()
|
||||
voice = voice or DEFAULT_AZURE_VOICE
|
||||
ssml = (
|
||||
f"<speak version='1.0' xml:lang='sr-RS'>"
|
||||
f"<voice name='{voice}'>{escape(text)}</voice></speak>"
|
||||
)
|
||||
url = (
|
||||
f"https://{s.azure_speech_region}.tts.speech.microsoft.com/"
|
||||
"cognitiveservices/v1"
|
||||
)
|
||||
async with httpx.AsyncClient(timeout=15) as client:
|
||||
resp = await client.post(
|
||||
url,
|
||||
content=ssml.encode(),
|
||||
headers={
|
||||
"Ocp-Apim-Subscription-Key": s.azure_speech_key,
|
||||
"Content-Type": "application/ssml+xml",
|
||||
"X-Microsoft-OutputFormat": "raw-8khz-16bit-mono-pcm",
|
||||
"User-Agent": "gogotelefon",
|
||||
},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.content
|
||||
|
||||
|
||||
class ElevenLabsTTS:
|
||||
async def synthesize(self, text: str, voice: str = "") -> bytes:
|
||||
s = get_settings()
|
||||
voice_id = voice or "JBFqnCBsd6RMkjVDRZzb"
|
||||
async with httpx.AsyncClient(timeout=20) as client:
|
||||
resp = await client.post(
|
||||
f"https://api.elevenlabs.io/v1/text-to-speech/{voice_id}",
|
||||
params={"output_format": "pcm_16000"},
|
||||
headers={"xi-api-key": s.elevenlabs_api_key},
|
||||
json={"text": text, "model_id": "eleven_flash_v2_5"},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resample(resp.content, 16000, SAMPLE_RATE)
|
||||
|
||||
|
||||
class FakeTTS:
|
||||
"""Test fake: N ms of tone per character (deterministic, VAD-audible)."""
|
||||
|
||||
def __init__(self, ms_per_char: int = 5):
|
||||
self.ms_per_char = ms_per_char
|
||||
self.spoken: list[str] = []
|
||||
|
||||
async def synthesize(self, text: str, voice: str = "") -> bytes:
|
||||
self.spoken.append(text)
|
||||
return tone(440, max(40, self.ms_per_char * len(text)))
|
||||
|
||||
|
||||
def make_tts(provider_and_voice: str = "") -> tuple[TTSProvider, str]:
|
||||
"""Resolve a tenant voice id 'provider:voice' → (provider instance, voice)."""
|
||||
s = get_settings()
|
||||
provider_name, _, voice = (provider_and_voice or "").partition(":")
|
||||
if not voice:
|
||||
provider_name, voice = s.tts_provider, ""
|
||||
if provider_name == "elevenlabs":
|
||||
return ElevenLabsTTS(), voice
|
||||
return AzureTTS(), voice or DEFAULT_AZURE_VOICE
|
||||
|
||||
|
||||
_tts: TTSProvider | None = None
|
||||
|
||||
|
||||
def get_tts(tenant_voice: str = "") -> tuple[TTSProvider, str]:
|
||||
if _tts is not None: # test override
|
||||
_, _, voice = (tenant_voice or "").partition(":")
|
||||
return _tts, voice
|
||||
return make_tts(tenant_voice)
|
||||
|
||||
|
||||
def set_tts(provider: TTSProvider | None) -> None:
|
||||
global _tts
|
||||
_tts = provider
|
||||
83
gogo/voice/vad.py
Normal file
83
gogo/voice/vad.py
Normal file
@@ -0,0 +1,83 @@
|
||||
"""Utterance detection: adaptive energy VAD with hangover.
|
||||
|
||||
Deliberately simple and dependency-free; swap for silero/webrtcvad later if the
|
||||
Phase-0 PoC shows it is needed. Operates on 20 ms SLIN frames.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from gogo.voice.audio import FRAME_MS, rms
|
||||
|
||||
|
||||
@dataclass
|
||||
class VadConfig:
|
||||
# energy threshold = max(min_threshold, noise_floor * ratio)
|
||||
min_threshold: int = 500
|
||||
noise_ratio: float = 3.0
|
||||
start_ms: int = 120 # this much voiced audio starts an utterance
|
||||
end_ms: int = 700 # this much silence ends it
|
||||
max_utterance_ms: int = 15000 # hard cap
|
||||
pre_roll_ms: int = 200 # audio kept from before the trigger
|
||||
|
||||
|
||||
@dataclass
|
||||
class UtteranceCollector:
|
||||
"""Feed 20 ms frames; returns the full utterance PCM when one completes."""
|
||||
|
||||
config: VadConfig = field(default_factory=VadConfig)
|
||||
_noise_floor: float = 200.0
|
||||
_voiced_ms: int = 0
|
||||
_silence_ms: int = 0
|
||||
_in_utterance: bool = False
|
||||
_buffer: bytearray = field(default_factory=bytearray)
|
||||
_pre_roll: bytearray = field(default_factory=bytearray)
|
||||
|
||||
def is_speech(self, frame: bytes) -> bool:
|
||||
energy = rms(frame)
|
||||
threshold = max(self.config.min_threshold, self._noise_floor * self.config.noise_ratio)
|
||||
speech = energy > threshold
|
||||
if not speech:
|
||||
# slowly track the noise floor on non-speech frames
|
||||
self._noise_floor = 0.95 * self._noise_floor + 0.05 * energy
|
||||
return speech
|
||||
|
||||
def feed(self, frame: bytes) -> bytes | None:
|
||||
"""Returns utterance PCM when a complete utterance is detected, else None."""
|
||||
speech = self.is_speech(frame)
|
||||
|
||||
if not self._in_utterance:
|
||||
self._pre_roll.extend(frame)
|
||||
max_pre = self.config.pre_roll_ms * len(frame) // FRAME_MS
|
||||
if len(self._pre_roll) > max_pre:
|
||||
del self._pre_roll[: len(self._pre_roll) - max_pre]
|
||||
if speech:
|
||||
self._voiced_ms += FRAME_MS
|
||||
if self._voiced_ms >= self.config.start_ms:
|
||||
self._in_utterance = True
|
||||
self._buffer = bytearray(self._pre_roll)
|
||||
self._silence_ms = 0
|
||||
else:
|
||||
self._voiced_ms = 0
|
||||
return None
|
||||
|
||||
self._buffer.extend(frame)
|
||||
if speech:
|
||||
self._silence_ms = 0
|
||||
else:
|
||||
self._silence_ms += FRAME_MS
|
||||
|
||||
utterance_ms = len(self._buffer) * FRAME_MS // len(frame)
|
||||
if self._silence_ms >= self.config.end_ms or utterance_ms >= self.config.max_utterance_ms:
|
||||
utterance = bytes(self._buffer)
|
||||
self.reset()
|
||||
return utterance
|
||||
return None
|
||||
|
||||
def reset(self) -> None:
|
||||
self._in_utterance = False
|
||||
self._voiced_ms = 0
|
||||
self._silence_ms = 0
|
||||
self._buffer = bytearray()
|
||||
self._pre_roll = bytearray()
|
||||
Reference in New Issue
Block a user