feat: Add Piper TTS integration with pitch control support
- Created PiperTTSService with pitch shifting (-12 to +12 semitones) - Created UnifiedTTSService to manage multiple TTS engines - Added engine selection endpoints (GET/POST /api/tts/engines) - Added voices listing endpoint (GET /api/tts/voices) - Added pitch_adjustments parameter to TTS generation - Created install_piper.sh for easy Piper installation - Updated backend README with multi-engine documentation - Created comprehensive PIPER_TTS_GUIDE.md - Added TTS_ENGINE environment variable for engine selection - Updated requirements.txt with piper-tts and pydub This allows users to choose between Kokoro (high quality) and Piper (fast with pitch control) TTS engines at runtime.
This commit is contained in:
219
backend/services/piper_tts_service.py
Normal file
219
backend/services/piper_tts_service.py
Normal file
@@ -0,0 +1,219 @@
|
||||
"""
|
||||
Piper TTS service for generating multi-voice audiobooks.
|
||||
"""
|
||||
import logging
|
||||
import io
|
||||
import os
|
||||
import subprocess
|
||||
from typing import Dict, List, Tuple, Optional
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
from pydub import AudioSegment
|
||||
from pydub.effects import speedup
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PiperTTSService:
|
||||
"""
|
||||
Service for text-to-speech synthesis using Piper TTS.
|
||||
|
||||
Piper is a fast, local neural TTS system that supports multiple voices
|
||||
and allows pitch modification through audio processing.
|
||||
"""
|
||||
|
||||
# Default voice for unmapped roles
|
||||
DEFAULT_VOICE = "en_US-lessac-medium"
|
||||
|
||||
# Sample rate for Piper TTS (typically 22050)
|
||||
SAMPLE_RATE = 22050
|
||||
|
||||
# Available Piper voices with their characteristics
|
||||
AVAILABLE_VOICES = {
|
||||
# English voices
|
||||
"en_US-lessac-medium": {"lang": "en", "gender": "male", "quality": "medium"},
|
||||
"en_US-libritts-high": {"lang": "en", "gender": "neutral", "quality": "high"},
|
||||
"en_US-amy-medium": {"lang": "en", "gender": "female", "quality": "medium"},
|
||||
"en_GB-alan-medium": {"lang": "en", "gender": "male", "quality": "medium"},
|
||||
"en_GB-alba-medium": {"lang": "en", "gender": "female", "quality": "medium"},
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
"""Initializes Piper TTS service."""
|
||||
self.models_dir = os.path.join(os.path.dirname(__file__), "..", "piper_models")
|
||||
os.makedirs(self.models_dir, exist_ok=True)
|
||||
logger.info("PiperTTSService initialized")
|
||||
|
||||
def _check_piper_installed(self) -> bool:
|
||||
"""Check if Piper is installed and accessible."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["piper", "--version"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5
|
||||
)
|
||||
return result.returncode == 0
|
||||
except (subprocess.SubprocessError, FileNotFoundError):
|
||||
return False
|
||||
|
||||
def _download_voice_model(self, voice: str) -> bool:
|
||||
"""
|
||||
Download a Piper voice model if not already present.
|
||||
|
||||
Args:
|
||||
voice: Voice ID to download
|
||||
|
||||
Returns:
|
||||
True if model is available, False otherwise
|
||||
"""
|
||||
model_path = os.path.join(self.models_dir, f"{voice}.onnx")
|
||||
config_path = os.path.join(self.models_dir, f"{voice}.onnx.json")
|
||||
|
||||
# Check if model already exists
|
||||
if os.path.exists(model_path) and os.path.exists(config_path):
|
||||
return True
|
||||
|
||||
logger.info(f"Downloading Piper voice model: {voice}")
|
||||
|
||||
# Download using piper's built-in download
|
||||
try:
|
||||
subprocess.run(
|
||||
["piper", "--model", voice, "--download-dir", self.models_dir],
|
||||
check=True,
|
||||
timeout=60
|
||||
)
|
||||
return os.path.exists(model_path)
|
||||
except subprocess.SubprocessError as e:
|
||||
logger.error(f"Failed to download voice model {voice}: {e}")
|
||||
return False
|
||||
|
||||
def synthesize_with_pitch(
|
||||
self,
|
||||
text: str,
|
||||
voice: str,
|
||||
speed: float = 1.0,
|
||||
pitch_shift: int = 0
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
Synthesizes text with Piper and applies pitch shifting.
|
||||
|
||||
Args:
|
||||
text: Text to synthesize
|
||||
voice: Piper voice ID
|
||||
speed: Playback speed (0.5-2.0)
|
||||
pitch_shift: Semitones to shift pitch (-12 to +12)
|
||||
|
||||
Returns:
|
||||
Audio samples as numpy array
|
||||
"""
|
||||
# Ensure voice model is available
|
||||
if not self._download_voice_model(voice):
|
||||
logger.warning(f"Voice {voice} not available, using default")
|
||||
voice = self.DEFAULT_VOICE
|
||||
self._download_voice_model(voice)
|
||||
|
||||
model_path = os.path.join(self.models_dir, f"{voice}.onnx")
|
||||
|
||||
# Generate audio with Piper
|
||||
try:
|
||||
# Run piper command
|
||||
process = subprocess.Popen(
|
||||
["piper", "--model", model_path, "--output-raw"],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE
|
||||
)
|
||||
|
||||
# Send text and get audio
|
||||
stdout, stderr = process.communicate(input=text.encode('utf-8'), timeout=30)
|
||||
|
||||
if process.returncode != 0:
|
||||
logger.error(f"Piper synthesis failed: {stderr.decode()}")
|
||||
return np.array([], dtype=np.float32)
|
||||
|
||||
# Convert raw audio to numpy array
|
||||
audio_data = np.frombuffer(stdout, dtype=np.int16)
|
||||
audio_float = audio_data.astype(np.float32) / 32768.0
|
||||
|
||||
# Apply pitch shift and speed change using pydub
|
||||
if pitch_shift != 0 or speed != 1.0:
|
||||
audio_float = self._apply_audio_effects(
|
||||
audio_float,
|
||||
self.SAMPLE_RATE,
|
||||
speed,
|
||||
pitch_shift
|
||||
)
|
||||
|
||||
return audio_float
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.error("Piper synthesis timed out")
|
||||
return np.array([], dtype=np.float32)
|
||||
except Exception as e:
|
||||
logger.error(f"Error in Piper synthesis: {e}")
|
||||
return np.array([], dtype=np.float32)
|
||||
|
||||
def _apply_audio_effects(
|
||||
self,
|
||||
audio: np.ndarray,
|
||||
sample_rate: int,
|
||||
speed: float,
|
||||
pitch_shift: int
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
Apply speed and pitch effects to audio using pydub.
|
||||
|
||||
Args:
|
||||
audio: Audio samples
|
||||
sample_rate: Sample rate
|
||||
speed: Speed multiplier
|
||||
pitch_shift: Semitones to shift
|
||||
|
||||
Returns:
|
||||
Processed audio samples
|
||||
"""
|
||||
try:
|
||||
# Convert numpy to pydub AudioSegment
|
||||
audio_int16 = (audio * 32768.0).astype(np.int16)
|
||||
audio_bytes = audio_int16.tobytes()
|
||||
|
||||
audio_segment = AudioSegment(
|
||||
audio_bytes,
|
||||
frame_rate=sample_rate,
|
||||
sample_width=2,
|
||||
channels=1
|
||||
)
|
||||
|
||||
# Apply pitch shift
|
||||
if pitch_shift != 0:
|
||||
# Pitch shift by changing frame rate then resampling
|
||||
new_sample_rate = int(sample_rate * (2.0 ** (pitch_shift / 12.0)))
|
||||
audio_segment = audio_segment._spawn(
|
||||
audio_segment.raw_data,
|
||||
overrides={'frame_rate': new_sample_rate}
|
||||
)
|
||||
audio_segment = audio_segment.set_frame_rate(sample_rate)
|
||||
|
||||
# Apply speed change
|
||||
if speed != 1.0:
|
||||
audio_segment = speedup(audio_segment, playback_speed=speed)
|
||||
|
||||
# Convert back to numpy
|
||||
samples = np.array(audio_segment.get_array_of_samples())
|
||||
audio_float = samples.astype(np.float32) / 32768.0
|
||||
|
||||
return audio_float
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error applying audio effects: {e}")
|
||||
return audio
|
||||
|
||||
def get_available_voices(self) -> Dict[str, Dict[str, str]]:
|
||||
"""
|
||||
Returns dictionary of available Piper voices.
|
||||
|
||||
Returns:
|
||||
Dict mapping voice IDs to their characteristics
|
||||
"""
|
||||
return self.AVAILABLE_VOICES.copy()
|
||||
283
backend/services/unified_tts_service.py
Normal file
283
backend/services/unified_tts_service.py
Normal file
@@ -0,0 +1,283 @@
|
||||
"""
|
||||
Unified TTS service that supports multiple TTS engines (Kokoro, Piper).
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
from typing import Dict, List, Tuple, Optional
|
||||
from enum import Enum
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TTSEngine(str, Enum):
|
||||
"""Available TTS engines."""
|
||||
KOKORO = "kokoro"
|
||||
PIPER = "piper"
|
||||
|
||||
|
||||
class UnifiedTTSService:
|
||||
"""
|
||||
Unified service that delegates to specific TTS engines.
|
||||
Allows runtime selection between Kokoro and Piper TTS.
|
||||
"""
|
||||
|
||||
def __init__(self, engine: Optional[str] = None, ollama_service=None):
|
||||
"""
|
||||
Initialize unified TTS service.
|
||||
|
||||
Args:
|
||||
engine: TTS engine to use ('kokoro' or 'piper').
|
||||
Defaults to TTS_ENGINE env var or 'kokoro'
|
||||
ollama_service: Optional OllamaService for AI segmentation
|
||||
"""
|
||||
self.engine_name = engine or os.getenv("TTS_ENGINE", "kokoro")
|
||||
self.ollama_service = ollama_service
|
||||
|
||||
# Lazy load engines
|
||||
self._kokoro_service = None
|
||||
self._piper_service = None
|
||||
|
||||
logger.info(f"UnifiedTTSService initialized with engine: {self.engine_name}")
|
||||
|
||||
@property
|
||||
def kokoro_service(self):
|
||||
"""Lazy load Kokoro service."""
|
||||
if self._kokoro_service is None:
|
||||
from .tts_service import TTSService
|
||||
self._kokoro_service = TTSService(self.ollama_service)
|
||||
return self._kokoro_service
|
||||
|
||||
@property
|
||||
def piper_service(self):
|
||||
"""Lazy load Piper service."""
|
||||
if self._piper_service is None:
|
||||
from .piper_tts_service import PiperTTSService
|
||||
self._piper_service = PiperTTSService()
|
||||
return self._piper_service
|
||||
|
||||
def set_engine(self, engine: str):
|
||||
"""
|
||||
Change the active TTS engine.
|
||||
|
||||
Args:
|
||||
engine: Engine name ('kokoro' or 'piper')
|
||||
"""
|
||||
if engine not in [e.value for e in TTSEngine]:
|
||||
raise ValueError(f"Unknown engine: {engine}. Use 'kokoro' or 'piper'")
|
||||
|
||||
self.engine_name = engine
|
||||
logger.info(f"Switched TTS engine to: {engine}")
|
||||
|
||||
def get_available_engines(self) -> List[str]:
|
||||
"""
|
||||
Get list of available TTS engines.
|
||||
|
||||
Returns:
|
||||
List of engine names
|
||||
"""
|
||||
return [e.value for e in TTSEngine]
|
||||
|
||||
def get_available_voices(self) -> Dict[str, List[str]]:
|
||||
"""
|
||||
Get available voices for all engines.
|
||||
|
||||
Returns:
|
||||
Dict mapping engine names to lists of voice IDs
|
||||
"""
|
||||
voices = {}
|
||||
|
||||
# Kokoro voices (from documentation)
|
||||
voices["kokoro"] = [
|
||||
"af_bella", "af_nicole", "af_sarah", "af_sky",
|
||||
"am_adam", "am_michael", "bf_emma", "bf_isabella",
|
||||
"bm_george", "bm_lewis"
|
||||
]
|
||||
|
||||
# Piper voices
|
||||
try:
|
||||
piper_voices = self.piper_service.get_available_voices()
|
||||
voices["piper"] = list(piper_voices.keys())
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not load Piper voices: {e}")
|
||||
voices["piper"] = []
|
||||
|
||||
return voices
|
||||
|
||||
async def generate_script_segments(
|
||||
self,
|
||||
text: str,
|
||||
roles: List[str]
|
||||
) -> List[Tuple[str, str]]:
|
||||
"""
|
||||
Generate script segments with speaker assignments.
|
||||
Uses AI when available, falls back to pattern matching.
|
||||
|
||||
Args:
|
||||
text: Input text to segment
|
||||
roles: List of detected role IDs/names
|
||||
|
||||
Returns:
|
||||
List of tuples: (speaker_name, text_content)
|
||||
"""
|
||||
# This is engine-agnostic, use Kokoro's implementation
|
||||
return await self.kokoro_service.generate_script_segments(text, roles)
|
||||
|
||||
async def generate_audiobook(
|
||||
self,
|
||||
text: str,
|
||||
lang: str,
|
||||
role_voice_map: Dict[str, str],
|
||||
speed: float = 1.0,
|
||||
pitch_adjustments: Optional[Dict[str, int]] = None
|
||||
) -> bytes:
|
||||
"""
|
||||
Generate audiobook using the selected TTS engine.
|
||||
|
||||
Args:
|
||||
text: Input text
|
||||
lang: Language code
|
||||
role_voice_map: Role to voice ID mapping
|
||||
speed: Playback speed
|
||||
pitch_adjustments: Optional dict of role to pitch shift (semitones)
|
||||
|
||||
Returns:
|
||||
Audio data as bytes (WAV format)
|
||||
"""
|
||||
if self.engine_name == TTSEngine.KOKORO:
|
||||
# Kokoro doesn't support pitch adjustment
|
||||
return await self.kokoro_service.generate_audiobook(
|
||||
text, lang, role_voice_map, speed
|
||||
)
|
||||
|
||||
elif self.engine_name == TTSEngine.PIPER:
|
||||
# Use Piper with pitch support
|
||||
return await self._generate_with_piper(
|
||||
text, lang, role_voice_map, speed, pitch_adjustments or {}
|
||||
)
|
||||
|
||||
else:
|
||||
raise ValueError(f"Unknown engine: {self.engine_name}")
|
||||
|
||||
async def generate_audiobook_from_segments(
|
||||
self,
|
||||
segments: List[Tuple[str, str]],
|
||||
lang: str,
|
||||
role_voice_map: Dict[str, str],
|
||||
speed: float = 1.0,
|
||||
pitch_adjustments: Optional[Dict[str, int]] = None
|
||||
) -> bytes:
|
||||
"""
|
||||
Generate audiobook from pre-segmented script.
|
||||
|
||||
Args:
|
||||
segments: List of (speaker_name, text_content) tuples
|
||||
lang: Language code
|
||||
role_voice_map: Role to voice ID mapping
|
||||
speed: Playback speed
|
||||
pitch_adjustments: Optional dict of role to pitch shift (semitones)
|
||||
|
||||
Returns:
|
||||
Audio data as bytes (WAV format)
|
||||
"""
|
||||
if self.engine_name == TTSEngine.KOKORO:
|
||||
return await self.kokoro_service.generate_audiobook_from_segments(
|
||||
segments, lang, role_voice_map, speed
|
||||
)
|
||||
|
||||
elif self.engine_name == TTSEngine.PIPER:
|
||||
return await self._generate_from_segments_with_piper(
|
||||
segments, lang, role_voice_map, speed, pitch_adjustments or {}
|
||||
)
|
||||
|
||||
else:
|
||||
raise ValueError(f"Unknown engine: {self.engine_name}")
|
||||
|
||||
async def _generate_with_piper(
|
||||
self,
|
||||
text: str,
|
||||
lang: str,
|
||||
role_voice_map: Dict[str, str],
|
||||
speed: float,
|
||||
pitch_adjustments: Dict[str, int]
|
||||
) -> bytes:
|
||||
"""Generate audiobook using Piper TTS with pitch control."""
|
||||
import io
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
|
||||
# Segment text using Kokoro's logic
|
||||
segments = await self.kokoro_service.segment_text_with_ai(text, role_voice_map)
|
||||
|
||||
if segments is None:
|
||||
segments = self.kokoro_service.segment_text(text, role_voice_map)
|
||||
|
||||
if not segments:
|
||||
raise ValueError("No segments to synthesize")
|
||||
|
||||
# Synthesize each segment with Piper
|
||||
audio_segments = []
|
||||
for role_id, voice, content in segments:
|
||||
pitch_shift = pitch_adjustments.get(role_id, 0)
|
||||
logger.debug(
|
||||
f"Synthesizing with Piper: role='{role_id}', voice='{voice}', "
|
||||
f"pitch={pitch_shift}"
|
||||
)
|
||||
|
||||
audio = self.piper_service.synthesize_with_pitch(
|
||||
content, voice, speed, pitch_shift
|
||||
)
|
||||
audio_segments.append(audio)
|
||||
|
||||
# Concatenate
|
||||
final_audio = np.concatenate([s for s in audio_segments if len(s) > 0])
|
||||
|
||||
# Convert to WAV bytes
|
||||
buffer = io.BytesIO()
|
||||
sf.write(buffer, final_audio, self.piper_service.SAMPLE_RATE, format='WAV')
|
||||
buffer.seek(0)
|
||||
|
||||
logger.info(f"Generated Piper audiobook: {len(buffer.getvalue())} bytes")
|
||||
return buffer.read()
|
||||
|
||||
async def _generate_from_segments_with_piper(
|
||||
self,
|
||||
segments: List[Tuple[str, str]],
|
||||
lang: str,
|
||||
role_voice_map: Dict[str, str],
|
||||
speed: float,
|
||||
pitch_adjustments: Dict[str, int]
|
||||
) -> bytes:
|
||||
"""Generate audiobook from segments using Piper TTS."""
|
||||
import io
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
|
||||
if not segments:
|
||||
raise ValueError("No segments provided")
|
||||
|
||||
# Synthesize each segment
|
||||
audio_segments = []
|
||||
for speaker, text in segments:
|
||||
voice = role_voice_map.get(speaker, self.piper_service.DEFAULT_VOICE)
|
||||
pitch_shift = pitch_adjustments.get(speaker, 0)
|
||||
|
||||
logger.debug(
|
||||
f"Synthesizing segment: speaker='{speaker}', voice='{voice}', "
|
||||
f"pitch={pitch_shift}"
|
||||
)
|
||||
|
||||
audio = self.piper_service.synthesize_with_pitch(
|
||||
text, voice, speed, pitch_shift
|
||||
)
|
||||
audio_segments.append(audio)
|
||||
|
||||
# Concatenate
|
||||
final_audio = np.concatenate([s for s in audio_segments if len(s) > 0])
|
||||
|
||||
# Convert to WAV bytes
|
||||
buffer = io.BytesIO()
|
||||
sf.write(buffer, final_audio, self.piper_service.SAMPLE_RATE, format='WAV')
|
||||
buffer.seek(0)
|
||||
|
||||
logger.info(f"Generated Piper audiobook from segments: {len(buffer.getvalue())} bytes")
|
||||
return buffer.read()
|
||||
Reference in New Issue
Block a user