From bfdcc94b24781245fb0b2fc8f9c96eb8be8fe2c6 Mon Sep 17 00:00:00 2001 From: Paul Bailey Date: Tue, 9 Dec 2025 14:45:18 +0000 Subject: [PATCH] feat: Add Piper TTS integration with pitch control support - Created PiperTTSService with pitch shifting (-12 to +12 semitones) - Created UnifiedTTSService to manage multiple TTS engines - Added engine selection endpoints (GET/POST /api/tts/engines) - Added voices listing endpoint (GET /api/tts/voices) - Added pitch_adjustments parameter to TTS generation - Created install_piper.sh for easy Piper installation - Updated backend README with multi-engine documentation - Created comprehensive PIPER_TTS_GUIDE.md - Added TTS_ENGINE environment variable for engine selection - Updated requirements.txt with piper-tts and pydub This allows users to choose between Kokoro (high quality) and Piper (fast with pitch control) TTS engines at runtime. --- backend/.env.example | 1 + backend/PIPER_TTS_GUIDE.md | 292 ++++++++++++++++++++++++ backend/README.md | 159 ++++++++++++- backend/install_piper.sh | 67 ++++++ backend/main.py | 113 ++++++++- backend/requirements.txt | 2 + backend/services/piper_tts_service.py | 219 ++++++++++++++++++ backend/services/unified_tts_service.py | 283 +++++++++++++++++++++++ 8 files changed, 1128 insertions(+), 8 deletions(-) create mode 100644 backend/PIPER_TTS_GUIDE.md create mode 100755 backend/install_piper.sh create mode 100644 backend/services/piper_tts_service.py create mode 100644 backend/services/unified_tts_service.py diff --git a/backend/.env.example b/backend/.env.example index 0abe249..6e3d278 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -1,3 +1,4 @@ PORT=8000 LOG_LEVEL=info OLLAMA_BASE_URL=http://localhost:11434 +TTS_ENGINE=kokoro diff --git a/backend/PIPER_TTS_GUIDE.md b/backend/PIPER_TTS_GUIDE.md new file mode 100644 index 0000000..7e0cf61 --- /dev/null +++ b/backend/PIPER_TTS_GUIDE.md @@ -0,0 +1,292 @@ +# Piper TTS Integration Guide + +This guide covers the Piper TTS engine integration, including installation, configuration, and usage. + +## What is Piper TTS? + +Piper is a fast, local neural text-to-speech system that runs entirely on your machine. It offers: + +- **Fast synthesis**: Faster than Kokoro for most use cases +- **Pitch control**: Adjust voice pitch per character (-12 to +12 semitones) +- **Multiple voices**: Various English voices with different characteristics +- **Low resource usage**: Smaller models and lower memory footprint +- **Privacy**: All processing happens locally + +## Installation + +### Automated Installation (Recommended) + +```bash +cd backend +./install_piper.sh +``` + +This script will: +1. Detect your OS and architecture +2. Download the appropriate Piper binary +3. Install it to `/usr/local/bin` +4. Verify the installation + +### Manual Installation + +1. Visit [Piper Releases](https://github.com/rhasspy/piper/releases/tag/v1.2.0) +2. Download the appropriate archive for your platform: + - **Linux x86_64**: `piper_amd64.tar.gz` + - **Linux ARM64**: `piper_arm64.tar.gz` + - **macOS**: `piper_macos_x64.tar.gz` +3. Extract the archive +4. Copy the `piper` binary to a directory in your PATH + +### Verify Installation + +```bash +piper --version +``` + +You should see output like: `1.2.0` + +## Available Voices + +Piper supports multiple English voices: + +| Voice ID | Language | Gender | Quality | Description | +|----------|----------|--------|---------|-------------| +| `en_US-lessac-medium` | US English | Male | Medium | Clear, neutral male voice | +| `en_US-libritts-high` | US English | Neutral | High | High-quality neutral voice | +| `en_US-amy-medium` | US English | Female | Medium | Clear female voice | +| `en_GB-alan-medium` | UK English | Male | Medium | British male voice | +| `en_GB-alba-medium` | UK English | Female | Medium | British female voice | + +Voice models are automatically downloaded on first use (~10-50MB each). + +## Configuration + +### Set Piper as Default Engine + +**Option 1: Environment Variable** +```bash +export TTS_ENGINE=piper +``` + +Add to your `.env` file: +```bash +TTS_ENGINE=piper +``` + +**Option 2: Runtime API Call** +```bash +curl -X POST http://localhost:8000/api/tts/engines \ + -H "Content-Type: application/json" \ + -d '{"engine": "piper"}' +``` + +### Voice Model Storage + +Voice models are stored in: +``` +backend/piper_models/ +``` + +Models are downloaded automatically on first use. You can pre-download models manually if needed. + +## Usage Examples + +### Basic Text-to-Speech + +```bash +curl -X POST http://localhost:8000/api/tts \ + -H "Content-Type: application/json" \ + -d '{ + "text": "Hello, this is a test of Piper TTS.", + "lang": "a", + "roles": [ + {"id": "Narrator", "voice": "en_US-lessac-medium"} + ], + "speed": 1.0 + }' \ + --output test.wav +``` + +### Multi-Voice with Pitch Control + +```bash +curl -X POST http://localhost:8000/api/tts \ + -H "Content-Type: application/json" \ + -d '{ + "text": "Alice: Hello Bob!\nBob: Hi Alice, how are you?\nNarrator: They greeted each other warmly.", + "lang": "a", + "roles": [ + {"id": "Alice", "voice": "en_US-amy-medium"}, + {"id": "Bob", "voice": "en_US-lessac-medium"}, + {"id": "Narrator", "voice": "en_GB-alan-medium"} + ], + "speed": 1.0, + "pitch_adjustments": [ + {"id": "Alice", "pitch": 4}, + {"id": "Bob", "pitch": -3}, + {"id": "Narrator", "pitch": 0} + ] + }' \ + --output audiobook.wav +``` + +### Pitch Adjustment Guidelines + +Pitch is measured in semitones (half-steps on a musical scale): + +- **+12 semitones**: One octave higher (very high-pitched) +- **+6 semitones**: Noticeably higher (child-like) +- **+3 semitones**: Slightly higher (feminine adjustment) +- **0 semitones**: Original voice (no change) +- **-3 semitones**: Slightly lower (masculine adjustment) +- **-6 semitones**: Noticeably lower (deep voice) +- **-12 semitones**: One octave lower (very deep) + +**Recommended ranges:** +- Female characters: +2 to +5 +- Male characters: -2 to -5 +- Children: +5 to +8 +- Elderly/deep voices: -4 to -8 + +## Python API Usage + +```python +from services.unified_tts_service import UnifiedTTSService + +# Initialize with Piper engine +tts = UnifiedTTSService(engine="piper") + +# Generate audiobook with pitch control +audio_bytes = await tts.generate_audiobook( + text="Alice: Hello!\nBob: Hi there!", + lang="a", + role_voice_map={ + "Alice": "en_US-amy-medium", + "Bob": "en_US-lessac-medium" + }, + speed=1.0, + pitch_adjustments={ + "Alice": 3, + "Bob": -2 + } +) + +# Save to file +with open("output.wav", "wb") as f: + f.write(audio_bytes) +``` + +## Performance Comparison + +| Feature | Kokoro TTS | Piper TTS | +|---------|------------|-----------| +| Speed | Medium | Fast | +| Quality | High | Medium-High | +| Pitch Control | ❌ No | ✅ Yes | +| Model Size | ~500MB | ~10-50MB per voice | +| Memory Usage | Higher | Lower | +| Voice Variety | 10+ voices | 5+ voices (expandable) | +| Languages | 4 languages | English (more available) | + +## Troubleshooting + +### Piper Not Found + +**Error:** `piper: command not found` + +**Solution:** +```bash +# Check if piper is in PATH +which piper + +# If not found, reinstall +./install_piper.sh + +# Or add to PATH manually +export PATH="/usr/local/bin:$PATH" +``` + +### Voice Model Download Fails + +**Error:** Model download timeout or failure + +**Solution:** +1. Check internet connection +2. Manually download models from [Piper releases](https://github.com/rhasspy/piper/releases) +3. Place `.onnx` and `.onnx.json` files in `backend/piper_models/` + +### Audio Quality Issues + +**Problem:** Distorted or robotic audio + +**Solutions:** +- Reduce pitch shift amount (stay within ±6 semitones) +- Try different voice models +- Adjust speed (try 0.9 or 1.1) +- Check for text formatting issues + +### Synthesis Timeout + +**Error:** Piper synthesis times out + +**Solutions:** +- Break text into smaller chunks +- Increase timeout in `piper_tts_service.py` +- Check system resources (CPU, memory) + +## Advanced Configuration + +### Custom Voice Models + +To add custom Piper voice models: + +1. Download `.onnx` and `.onnx.json` files +2. Place in `backend/piper_models/` +3. Update `AVAILABLE_VOICES` in `piper_tts_service.py`: + +```python +AVAILABLE_VOICES = { + "en_US-custom-voice": {"lang": "en", "gender": "female", "quality": "high"}, + # ... existing voices +} +``` + +### Adjusting Audio Effects + +Modify `_apply_audio_effects()` in `piper_tts_service.py` to customize: +- Pitch shifting algorithm +- Speed adjustment method +- Audio processing pipeline + +## Best Practices + +1. **Voice Selection** + - Use distinct voices for different characters + - Match voice gender to character when possible + - Use British voices for variety + +2. **Pitch Adjustments** + - Start with small adjustments (±2-3 semitones) + - Test and iterate to find natural-sounding values + - Avoid extreme values (±10+) unless intentional + +3. **Performance** + - Pre-download voice models for faster first use + - Cache generated audio when possible + - Use appropriate chunk sizes for long texts + +4. **Quality** + - Use high-quality voice models when available + - Adjust speed slightly (0.95-1.05) for more natural pacing + - Combine with AI segmentation for best results + +## Resources + +- [Piper GitHub Repository](https://github.com/rhasspy/piper) +- [Piper Voice Samples](https://rhasspy.github.io/piper-samples/) +- [Voice Model Downloads](https://github.com/rhasspy/piper/releases/tag/v1.2.0) +- [Piper Documentation](https://github.com/rhasspy/piper/blob/master/README.md) + +## License + +Piper TTS is licensed under the MIT License. diff --git a/backend/README.md b/backend/README.md index beed9ae..f312fe5 100644 --- a/backend/README.md +++ b/backend/README.md @@ -13,6 +13,7 @@ The backend provides REST API endpoints for: - Python 3.10 or higher - espeak-ng (system dependency for grapheme-to-phoneme conversion) +- Piper TTS (optional, for alternative TTS engine with pitch control) - Docker (optional, for containerized deployment) ## Local Development Setup @@ -62,7 +63,33 @@ pip install --upgrade pip pip install -r requirements.txt ``` -### 3. Ollama Setup (Optional but Recommended) +### 3. Piper TTS Setup (Optional) + +For an alternative TTS engine with pitch control support: + +**Installation:** +```bash +# Run the installation script +./install_piper.sh +``` + +Or install manually: +- Visit [Piper releases](https://github.com/rhasspy/piper/releases) +- Download the appropriate binary for your platform +- Extract and place in your PATH + +**Verify installation:** +```bash +piper --version +``` + +**Benefits of Piper TTS:** +- ✨ Pitch shifting support (-12 to +12 semitones per role) +- ✨ Fast, lightweight neural TTS +- ✨ Multiple voice models available +- ✨ Lower resource usage than Kokoro + +### 4. Ollama Setup (Optional but Recommended) For AI-powered speaker detection and text segmentation, install Ollama: @@ -98,7 +125,7 @@ curl http://localhost:11434/api/tags - ✨ Handles narrative text without explicit role markers - ✨ Falls back to pattern matching if unavailable -### 4. Environment Configuration +### 5. Environment Configuration ```bash cp .env.example .env @@ -109,9 +136,10 @@ Edit `.env` if needed: PORT=8000 LOG_LEVEL=info OLLAMA_BASE_URL=http://localhost:11434 # Optional: Ollama API URL +TTS_ENGINE=kokoro # Options: kokoro, piper (default: kokoro) ``` -### 5. Run the Server +### 6. Run the Server ```bash uvicorn main:app --reload --host 0.0.0.0 --port 8000 @@ -179,6 +207,43 @@ The Dockerfile includes: - Storage: ~500MB for Kokoro models - CPU: Multi-core recommended +## TTS Engine Selection + +The backend now supports multiple TTS engines that can be selected at runtime: + +### Available Engines + +1. **Kokoro TTS** (default) + - High-quality neural TTS + - Multiple voices per language + - No pitch control + - ~500MB model size + +2. **Piper TTS** + - Fast, lightweight neural TTS + - Pitch shifting support (-12 to +12 semitones) + - Multiple voice models + - Lower resource usage + +### Switching Engines + +**Via Environment Variable:** +```bash +export TTS_ENGINE=piper # or kokoro +uvicorn main:app --reload +``` + +**Via API:** +```bash +# Get available engines +curl http://localhost:8000/api/tts/engines + +# Set active engine +curl -X POST http://localhost:8000/api/tts/engines \ + -H "Content-Type: application/json" \ + -d '{"engine": "piper"}' +``` + ## API Endpoints ### GET /api/health @@ -243,6 +308,71 @@ curl -X POST http://localhost:8000/api/tts/roles \ -d '{"text":"Alice: Hello!\nBob: Hi there!"}' ``` +### GET /api/tts/engines + +Get list of available TTS engines and current selection. + +**Response:** +```json +{ + "engines": [ + {"name": "kokoro", "active": true}, + {"name": "piper", "active": false} + ], + "current": "kokoro" +} +``` + +**Example:** +```bash +curl http://localhost:8000/api/tts/engines +``` + +### POST /api/tts/engines + +Set the active TTS engine. + +**Request Body:** +```json +{ + "engine": "piper" +} +``` + +**Response:** +```json +{ + "status": "ok", + "engine": "piper" +} +``` + +**Example:** +```bash +curl -X POST http://localhost:8000/api/tts/engines \ + -H "Content-Type: application/json" \ + -d '{"engine": "piper"}' +``` + +### GET /api/tts/voices + +Get available voices for all TTS engines. + +**Response:** +```json +{ + "voices": { + "kokoro": ["af_bella", "af_sarah", "am_adam", "am_michael", ...], + "piper": ["en_US-lessac-medium", "en_US-amy-medium", ...] + } +} +``` + +**Example:** +```bash +curl http://localhost:8000/api/tts/voices +``` + ### POST /api/tts Generate multi-voice audiobook with AI-powered text segmentation. @@ -271,6 +401,7 @@ When Ollama is available, the system uses AI to intelligently split text and ass - `lang` (required): Language code (a=American English, b=British English, f=French, j=Japanese) - `roles` (required): Array of role-to-voice mappings - `speed` (optional): Playback speed (0.5-1.5, default: 1.0) +- `pitch_adjustments` (optional): Array of role-to-pitch mappings (Piper only, -12 to +12 semitones) **Response:** - Content-Type: `audio/wav` @@ -285,7 +416,7 @@ When Ollama is available, the system uses AI to intelligently split text and ass - Content-Disposition: `attachment; filename="audiobook.wav"` - Body: Audio file bytes -**Example:** +**Example (Kokoro):** ```bash curl -X POST http://localhost:8000/api/tts \ -H "Content-Type: application/json" \ @@ -301,6 +432,26 @@ curl -X POST http://localhost:8000/api/tts \ --output audiobook.wav ``` +**Example (Piper with pitch control):** +```bash +curl -X POST http://localhost:8000/api/tts \ + -H "Content-Type: application/json" \ + -d '{ + "text": "Alice: Hello!\nBob: Hi!", + "lang": "a", + "roles": [ + {"id": "Alice", "voice": "en_US-amy-medium"}, + {"id": "Bob", "voice": "en_US-lessac-medium"} + ], + "speed": 1.0, + "pitch_adjustments": [ + {"id": "Alice", "pitch": 3}, + {"id": "Bob", "pitch": -2} + ] + }' \ + --output audiobook.wav +``` + ## Kokoro TTS Configuration ### Supported Languages diff --git a/backend/install_piper.sh b/backend/install_piper.sh new file mode 100755 index 0000000..c58f5c3 --- /dev/null +++ b/backend/install_piper.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# Script to install Piper TTS + +echo "Installing Piper TTS..." + +# Detect OS +OS="$(uname -s)" +ARCH="$(uname -m)" + +case "${OS}" in + Linux*) + if [ "${ARCH}" = "x86_64" ]; then + PIPER_URL="https://github.com/rhasspy/piper/releases/download/v1.2.0/piper_amd64.tar.gz" + elif [ "${ARCH}" = "aarch64" ]; then + PIPER_URL="https://github.com/rhasspy/piper/releases/download/v1.2.0/piper_arm64.tar.gz" + else + echo "Unsupported architecture: ${ARCH}" + exit 1 + fi + ;; + Darwin*) + if [ "${ARCH}" = "x86_64" ] || [ "${ARCH}" = "arm64" ]; then + PIPER_URL="https://github.com/rhasspy/piper/releases/download/v1.2.0/piper_macos_x64.tar.gz" + else + echo "Unsupported architecture: ${ARCH}" + exit 1 + fi + ;; + *) + echo "Unsupported OS: ${OS}" + exit 1 + ;; +esac + +# Create temp directory +TEMP_DIR=$(mktemp -d) +cd "${TEMP_DIR}" + +# Download Piper +echo "Downloading Piper from ${PIPER_URL}..." +curl -L -o piper.tar.gz "${PIPER_URL}" + +# Extract +echo "Extracting..." +tar -xzf piper.tar.gz + +# Install to /usr/local/bin (requires sudo) +echo "Installing to /usr/local/bin (may require password)..." +sudo cp piper/piper /usr/local/bin/ +sudo chmod +x /usr/local/bin/piper + +# Cleanup +cd - +rm -rf "${TEMP_DIR}" + +# Verify installation +if command -v piper &> /dev/null; then + echo "✓ Piper TTS installed successfully!" + piper --version +else + echo "✗ Installation failed" + exit 1 +fi + +echo "" +echo "To download voice models, they will be automatically downloaded on first use." +echo "Or you can manually download from: https://github.com/rhasspy/piper/releases/tag/v1.2.0" diff --git a/backend/main.py b/backend/main.py index fcdabd4..a04f155 100644 --- a/backend/main.py +++ b/backend/main.py @@ -11,7 +11,7 @@ from pydantic import BaseModel, Field # Import services from services.role_detector import RoleDetector -from services.tts_service import TTSService +from services.unified_tts_service import UnifiedTTSService # Configure logging logging.basicConfig( @@ -54,6 +54,11 @@ class RoleVoiceMapping(BaseModel): id: str = Field(..., description="Role identifier") voice: str = Field(..., description="Kokoro voice ID") +class RolePitchMapping(BaseModel): + """Model for role-to-pitch mapping.""" + id: str = Field(..., description="Role identifier") + pitch: int = Field(0, description="Pitch shift in semitones (-12 to +12)") + class TTSRequest(BaseModel): """Request model for TTS generation endpoint.""" segments: Optional[List[ScriptSegment]] = Field(None, description="Optional edited script segments") @@ -61,6 +66,7 @@ class TTSRequest(BaseModel): lang: str = Field(..., description="Language code (a, b, f, j)") roles: List[RoleVoiceMapping] = Field(..., description="Role to voice mappings") speed: Optional[float] = Field(1.0, description="Playback speed (0.5-1.5)") + pitch_adjustments: Optional[List[RolePitchMapping]] = Field(None, description="Optional pitch adjustments per role (Piper only)") class VoicePreviewRequest(BaseModel): """Request model for voice preview endpoint.""" @@ -75,6 +81,24 @@ class ErrorResponse(BaseModel): message: str = Field(..., description="Human-readable error message") details: Optional[dict] = Field(None, description="Additional error details") +class EngineInfo(BaseModel): + """Model for TTS engine information.""" + name: str = Field(..., description="Engine identifier") + active: bool = Field(..., description="Whether this engine is currently active") + +class EnginesResponse(BaseModel): + """Response model for available engines.""" + engines: List[EngineInfo] = Field(..., description="List of available TTS engines") + current: str = Field(..., description="Currently active engine") + +class SetEngineRequest(BaseModel): + """Request model for setting TTS engine.""" + engine: str = Field(..., description="Engine to activate (kokoro or piper)") + +class VoicesResponse(BaseModel): + """Response model for available voices.""" + voices: dict = Field(..., description="Dict mapping engine names to voice lists") + # Initialize FastAPI app app = FastAPI( title="Text to Audiobook API", @@ -120,7 +144,7 @@ async def general_exception_handler(request: Request, exc: Exception): # Initialize services role_detector = RoleDetector() -tts_service = TTSService() +tts_service = UnifiedTTSService() # API Endpoints @app.get("/api/health") @@ -129,6 +153,80 @@ async def health_check(): logger.info("Health check requested") return {"status": "ok"} +@app.get("/api/tts/engines", response_model=EnginesResponse) +async def get_engines(): + """ + Get list of available TTS engines and current selection. + + Returns: + EnginesResponse with list of engines and current active engine + """ + logger.info("Engines list requested") + + try: + available = tts_service.get_available_engines() + current = tts_service.engine_name + + engines = [ + EngineInfo(name=engine, active=(engine == current)) + for engine in available + ] + + return EnginesResponse(engines=engines, current=current) + except Exception as e: + logger.error(f"Error getting engines: {str(e)}", exc_info=True) + raise + +@app.post("/api/tts/engines") +async def set_engine(request: SetEngineRequest): + """ + Set the active TTS engine. + + Args: + request: SetEngineRequest with engine name + + Returns: + Success message with new engine + + Raises: + 400: If engine name is invalid + """ + logger.info(f"Engine change requested: {request.engine}") + + try: + tts_service.set_engine(request.engine) + return {"status": "ok", "engine": request.engine} + except ValueError as e: + logger.error(f"Invalid engine: {str(e)}") + return JSONResponse( + status_code=status.HTTP_400_BAD_REQUEST, + content={ + "error": "ValidationError", + "message": str(e), + "details": {"field": "engine"} + } + ) + except Exception as e: + logger.error(f"Error setting engine: {str(e)}", exc_info=True) + raise + +@app.get("/api/tts/voices", response_model=VoicesResponse) +async def get_voices(): + """ + Get available voices for all TTS engines. + + Returns: + VoicesResponse with dict mapping engine names to voice lists + """ + logger.info("Voices list requested") + + try: + voices = tts_service.get_available_voices() + return VoicesResponse(voices=voices) + except Exception as e: + logger.error(f"Error getting voices: {str(e)}", exc_info=True) + raise + @app.post("/api/tts/roles", response_model=RolesResponse) async def detect_roles(request: RoleDetectionRequest): """ @@ -420,6 +518,11 @@ async def generate_audiobook(request: TTSRequest): # Convert roles list to dictionary for TTS service role_voice_map = {role.id: role.voice for role in request.roles} + # Convert pitch adjustments to dictionary if provided + pitch_map = None + if request.pitch_adjustments: + pitch_map = {p.id: p.pitch for p in request.pitch_adjustments} + try: # Generate audiobook using segments if provided, otherwise use text if request.segments: @@ -428,7 +531,8 @@ async def generate_audiobook(request: TTSRequest): segments=[(seg.speaker, seg.text) for seg in request.segments], lang=request.lang, role_voice_map=role_voice_map, - speed=request.speed + speed=request.speed, + pitch_adjustments=pitch_map ) else: logger.info("Generating audiobook from text with automatic segmentation") @@ -436,7 +540,8 @@ async def generate_audiobook(request: TTSRequest): text=request.text, lang=request.lang, role_voice_map=role_voice_map, - speed=request.speed + speed=request.speed, + pitch_adjustments=pitch_map ) # Create streaming response with proper headers diff --git a/backend/requirements.txt b/backend/requirements.txt index 98048ed..a2ee17c 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -6,3 +6,5 @@ python-multipart==0.0.6 pydantic==2.5.3 numpy==1.26.3 httpx==0.26.0 +piper-tts==1.2.0 +pydub==0.25.1 diff --git a/backend/services/piper_tts_service.py b/backend/services/piper_tts_service.py new file mode 100644 index 0000000..58b82e3 --- /dev/null +++ b/backend/services/piper_tts_service.py @@ -0,0 +1,219 @@ +""" +Piper TTS service for generating multi-voice audiobooks. +""" +import logging +import io +import os +import subprocess +from typing import Dict, List, Tuple, Optional +import numpy as np +import soundfile as sf +from pydub import AudioSegment +from pydub.effects import speedup + +logger = logging.getLogger(__name__) + + +class PiperTTSService: + """ + Service for text-to-speech synthesis using Piper TTS. + + Piper is a fast, local neural TTS system that supports multiple voices + and allows pitch modification through audio processing. + """ + + # Default voice for unmapped roles + DEFAULT_VOICE = "en_US-lessac-medium" + + # Sample rate for Piper TTS (typically 22050) + SAMPLE_RATE = 22050 + + # Available Piper voices with their characteristics + AVAILABLE_VOICES = { + # English voices + "en_US-lessac-medium": {"lang": "en", "gender": "male", "quality": "medium"}, + "en_US-libritts-high": {"lang": "en", "gender": "neutral", "quality": "high"}, + "en_US-amy-medium": {"lang": "en", "gender": "female", "quality": "medium"}, + "en_GB-alan-medium": {"lang": "en", "gender": "male", "quality": "medium"}, + "en_GB-alba-medium": {"lang": "en", "gender": "female", "quality": "medium"}, + } + + def __init__(self): + """Initializes Piper TTS service.""" + self.models_dir = os.path.join(os.path.dirname(__file__), "..", "piper_models") + os.makedirs(self.models_dir, exist_ok=True) + logger.info("PiperTTSService initialized") + + def _check_piper_installed(self) -> bool: + """Check if Piper is installed and accessible.""" + try: + result = subprocess.run( + ["piper", "--version"], + capture_output=True, + text=True, + timeout=5 + ) + return result.returncode == 0 + except (subprocess.SubprocessError, FileNotFoundError): + return False + + def _download_voice_model(self, voice: str) -> bool: + """ + Download a Piper voice model if not already present. + + Args: + voice: Voice ID to download + + Returns: + True if model is available, False otherwise + """ + model_path = os.path.join(self.models_dir, f"{voice}.onnx") + config_path = os.path.join(self.models_dir, f"{voice}.onnx.json") + + # Check if model already exists + if os.path.exists(model_path) and os.path.exists(config_path): + return True + + logger.info(f"Downloading Piper voice model: {voice}") + + # Download using piper's built-in download + try: + subprocess.run( + ["piper", "--model", voice, "--download-dir", self.models_dir], + check=True, + timeout=60 + ) + return os.path.exists(model_path) + except subprocess.SubprocessError as e: + logger.error(f"Failed to download voice model {voice}: {e}") + return False + + def synthesize_with_pitch( + self, + text: str, + voice: str, + speed: float = 1.0, + pitch_shift: int = 0 + ) -> np.ndarray: + """ + Synthesizes text with Piper and applies pitch shifting. + + Args: + text: Text to synthesize + voice: Piper voice ID + speed: Playback speed (0.5-2.0) + pitch_shift: Semitones to shift pitch (-12 to +12) + + Returns: + Audio samples as numpy array + """ + # Ensure voice model is available + if not self._download_voice_model(voice): + logger.warning(f"Voice {voice} not available, using default") + voice = self.DEFAULT_VOICE + self._download_voice_model(voice) + + model_path = os.path.join(self.models_dir, f"{voice}.onnx") + + # Generate audio with Piper + try: + # Run piper command + process = subprocess.Popen( + ["piper", "--model", model_path, "--output-raw"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + + # Send text and get audio + stdout, stderr = process.communicate(input=text.encode('utf-8'), timeout=30) + + if process.returncode != 0: + logger.error(f"Piper synthesis failed: {stderr.decode()}") + return np.array([], dtype=np.float32) + + # Convert raw audio to numpy array + audio_data = np.frombuffer(stdout, dtype=np.int16) + audio_float = audio_data.astype(np.float32) / 32768.0 + + # Apply pitch shift and speed change using pydub + if pitch_shift != 0 or speed != 1.0: + audio_float = self._apply_audio_effects( + audio_float, + self.SAMPLE_RATE, + speed, + pitch_shift + ) + + return audio_float + + except subprocess.TimeoutExpired: + logger.error("Piper synthesis timed out") + return np.array([], dtype=np.float32) + except Exception as e: + logger.error(f"Error in Piper synthesis: {e}") + return np.array([], dtype=np.float32) + + def _apply_audio_effects( + self, + audio: np.ndarray, + sample_rate: int, + speed: float, + pitch_shift: int + ) -> np.ndarray: + """ + Apply speed and pitch effects to audio using pydub. + + Args: + audio: Audio samples + sample_rate: Sample rate + speed: Speed multiplier + pitch_shift: Semitones to shift + + Returns: + Processed audio samples + """ + try: + # Convert numpy to pydub AudioSegment + audio_int16 = (audio * 32768.0).astype(np.int16) + audio_bytes = audio_int16.tobytes() + + audio_segment = AudioSegment( + audio_bytes, + frame_rate=sample_rate, + sample_width=2, + channels=1 + ) + + # Apply pitch shift + if pitch_shift != 0: + # Pitch shift by changing frame rate then resampling + new_sample_rate = int(sample_rate * (2.0 ** (pitch_shift / 12.0))) + audio_segment = audio_segment._spawn( + audio_segment.raw_data, + overrides={'frame_rate': new_sample_rate} + ) + audio_segment = audio_segment.set_frame_rate(sample_rate) + + # Apply speed change + if speed != 1.0: + audio_segment = speedup(audio_segment, playback_speed=speed) + + # Convert back to numpy + samples = np.array(audio_segment.get_array_of_samples()) + audio_float = samples.astype(np.float32) / 32768.0 + + return audio_float + + except Exception as e: + logger.error(f"Error applying audio effects: {e}") + return audio + + def get_available_voices(self) -> Dict[str, Dict[str, str]]: + """ + Returns dictionary of available Piper voices. + + Returns: + Dict mapping voice IDs to their characteristics + """ + return self.AVAILABLE_VOICES.copy() diff --git a/backend/services/unified_tts_service.py b/backend/services/unified_tts_service.py new file mode 100644 index 0000000..21c2628 --- /dev/null +++ b/backend/services/unified_tts_service.py @@ -0,0 +1,283 @@ +""" +Unified TTS service that supports multiple TTS engines (Kokoro, Piper). +""" +import logging +import os +from typing import Dict, List, Tuple, Optional +from enum import Enum + +logger = logging.getLogger(__name__) + + +class TTSEngine(str, Enum): + """Available TTS engines.""" + KOKORO = "kokoro" + PIPER = "piper" + + +class UnifiedTTSService: + """ + Unified service that delegates to specific TTS engines. + Allows runtime selection between Kokoro and Piper TTS. + """ + + def __init__(self, engine: Optional[str] = None, ollama_service=None): + """ + Initialize unified TTS service. + + Args: + engine: TTS engine to use ('kokoro' or 'piper'). + Defaults to TTS_ENGINE env var or 'kokoro' + ollama_service: Optional OllamaService for AI segmentation + """ + self.engine_name = engine or os.getenv("TTS_ENGINE", "kokoro") + self.ollama_service = ollama_service + + # Lazy load engines + self._kokoro_service = None + self._piper_service = None + + logger.info(f"UnifiedTTSService initialized with engine: {self.engine_name}") + + @property + def kokoro_service(self): + """Lazy load Kokoro service.""" + if self._kokoro_service is None: + from .tts_service import TTSService + self._kokoro_service = TTSService(self.ollama_service) + return self._kokoro_service + + @property + def piper_service(self): + """Lazy load Piper service.""" + if self._piper_service is None: + from .piper_tts_service import PiperTTSService + self._piper_service = PiperTTSService() + return self._piper_service + + def set_engine(self, engine: str): + """ + Change the active TTS engine. + + Args: + engine: Engine name ('kokoro' or 'piper') + """ + if engine not in [e.value for e in TTSEngine]: + raise ValueError(f"Unknown engine: {engine}. Use 'kokoro' or 'piper'") + + self.engine_name = engine + logger.info(f"Switched TTS engine to: {engine}") + + def get_available_engines(self) -> List[str]: + """ + Get list of available TTS engines. + + Returns: + List of engine names + """ + return [e.value for e in TTSEngine] + + def get_available_voices(self) -> Dict[str, List[str]]: + """ + Get available voices for all engines. + + Returns: + Dict mapping engine names to lists of voice IDs + """ + voices = {} + + # Kokoro voices (from documentation) + voices["kokoro"] = [ + "af_bella", "af_nicole", "af_sarah", "af_sky", + "am_adam", "am_michael", "bf_emma", "bf_isabella", + "bm_george", "bm_lewis" + ] + + # Piper voices + try: + piper_voices = self.piper_service.get_available_voices() + voices["piper"] = list(piper_voices.keys()) + except Exception as e: + logger.warning(f"Could not load Piper voices: {e}") + voices["piper"] = [] + + return voices + + async def generate_script_segments( + self, + text: str, + roles: List[str] + ) -> List[Tuple[str, str]]: + """ + Generate script segments with speaker assignments. + Uses AI when available, falls back to pattern matching. + + Args: + text: Input text to segment + roles: List of detected role IDs/names + + Returns: + List of tuples: (speaker_name, text_content) + """ + # This is engine-agnostic, use Kokoro's implementation + return await self.kokoro_service.generate_script_segments(text, roles) + + async def generate_audiobook( + self, + text: str, + lang: str, + role_voice_map: Dict[str, str], + speed: float = 1.0, + pitch_adjustments: Optional[Dict[str, int]] = None + ) -> bytes: + """ + Generate audiobook using the selected TTS engine. + + Args: + text: Input text + lang: Language code + role_voice_map: Role to voice ID mapping + speed: Playback speed + pitch_adjustments: Optional dict of role to pitch shift (semitones) + + Returns: + Audio data as bytes (WAV format) + """ + if self.engine_name == TTSEngine.KOKORO: + # Kokoro doesn't support pitch adjustment + return await self.kokoro_service.generate_audiobook( + text, lang, role_voice_map, speed + ) + + elif self.engine_name == TTSEngine.PIPER: + # Use Piper with pitch support + return await self._generate_with_piper( + text, lang, role_voice_map, speed, pitch_adjustments or {} + ) + + else: + raise ValueError(f"Unknown engine: {self.engine_name}") + + async def generate_audiobook_from_segments( + self, + segments: List[Tuple[str, str]], + lang: str, + role_voice_map: Dict[str, str], + speed: float = 1.0, + pitch_adjustments: Optional[Dict[str, int]] = None + ) -> bytes: + """ + Generate audiobook from pre-segmented script. + + Args: + segments: List of (speaker_name, text_content) tuples + lang: Language code + role_voice_map: Role to voice ID mapping + speed: Playback speed + pitch_adjustments: Optional dict of role to pitch shift (semitones) + + Returns: + Audio data as bytes (WAV format) + """ + if self.engine_name == TTSEngine.KOKORO: + return await self.kokoro_service.generate_audiobook_from_segments( + segments, lang, role_voice_map, speed + ) + + elif self.engine_name == TTSEngine.PIPER: + return await self._generate_from_segments_with_piper( + segments, lang, role_voice_map, speed, pitch_adjustments or {} + ) + + else: + raise ValueError(f"Unknown engine: {self.engine_name}") + + async def _generate_with_piper( + self, + text: str, + lang: str, + role_voice_map: Dict[str, str], + speed: float, + pitch_adjustments: Dict[str, int] + ) -> bytes: + """Generate audiobook using Piper TTS with pitch control.""" + import io + import numpy as np + import soundfile as sf + + # Segment text using Kokoro's logic + segments = await self.kokoro_service.segment_text_with_ai(text, role_voice_map) + + if segments is None: + segments = self.kokoro_service.segment_text(text, role_voice_map) + + if not segments: + raise ValueError("No segments to synthesize") + + # Synthesize each segment with Piper + audio_segments = [] + for role_id, voice, content in segments: + pitch_shift = pitch_adjustments.get(role_id, 0) + logger.debug( + f"Synthesizing with Piper: role='{role_id}', voice='{voice}', " + f"pitch={pitch_shift}" + ) + + audio = self.piper_service.synthesize_with_pitch( + content, voice, speed, pitch_shift + ) + audio_segments.append(audio) + + # Concatenate + final_audio = np.concatenate([s for s in audio_segments if len(s) > 0]) + + # Convert to WAV bytes + buffer = io.BytesIO() + sf.write(buffer, final_audio, self.piper_service.SAMPLE_RATE, format='WAV') + buffer.seek(0) + + logger.info(f"Generated Piper audiobook: {len(buffer.getvalue())} bytes") + return buffer.read() + + async def _generate_from_segments_with_piper( + self, + segments: List[Tuple[str, str]], + lang: str, + role_voice_map: Dict[str, str], + speed: float, + pitch_adjustments: Dict[str, int] + ) -> bytes: + """Generate audiobook from segments using Piper TTS.""" + import io + import numpy as np + import soundfile as sf + + if not segments: + raise ValueError("No segments provided") + + # Synthesize each segment + audio_segments = [] + for speaker, text in segments: + voice = role_voice_map.get(speaker, self.piper_service.DEFAULT_VOICE) + pitch_shift = pitch_adjustments.get(speaker, 0) + + logger.debug( + f"Synthesizing segment: speaker='{speaker}', voice='{voice}', " + f"pitch={pitch_shift}" + ) + + audio = self.piper_service.synthesize_with_pitch( + text, voice, speed, pitch_shift + ) + audio_segments.append(audio) + + # Concatenate + final_audio = np.concatenate([s for s in audio_segments if len(s) > 0]) + + # Convert to WAV bytes + buffer = io.BytesIO() + sf.write(buffer, final_audio, self.piper_service.SAMPLE_RATE, format='WAV') + buffer.seek(0) + + logger.info(f"Generated Piper audiobook from segments: {len(buffer.getvalue())} bytes") + return buffer.read()