Add voice preview and age category modifiers
Features: - Voice preview button to test voices before generation - Age category selection (young, teen, adult, old) - Age categories adjust voice speed automatically: - Young: 1.15x (faster, energetic) - Teen: 1.08x (slightly faster) - Adult: 1.0x (normal) - Old: 0.85x (slower, measured) - Preview caching to avoid regenerating same voice - Backend /api/tts/preview endpoint for voice samples - Frontend previewVoice API function This allows users to hear how each voice sounds and adjust character ages to make them sound younger or older.
This commit is contained in:
@@ -62,6 +62,13 @@ class TTSRequest(BaseModel):
|
|||||||
roles: List[RoleVoiceMapping] = Field(..., description="Role to voice mappings")
|
roles: List[RoleVoiceMapping] = Field(..., description="Role to voice mappings")
|
||||||
speed: Optional[float] = Field(1.0, description="Playback speed (0.5-1.5)")
|
speed: Optional[float] = Field(1.0, description="Playback speed (0.5-1.5)")
|
||||||
|
|
||||||
|
class VoicePreviewRequest(BaseModel):
|
||||||
|
"""Request model for voice preview endpoint."""
|
||||||
|
voice: str = Field(..., description="Kokoro voice ID to preview")
|
||||||
|
lang: str = Field(..., description="Language code (a, b, f, j)")
|
||||||
|
speed: Optional[float] = Field(1.0, description="Playback speed (0.5-1.5)")
|
||||||
|
text: Optional[str] = Field(None, description="Custom text to preview (uses default if not provided)")
|
||||||
|
|
||||||
class ErrorResponse(BaseModel):
|
class ErrorResponse(BaseModel):
|
||||||
"""Standard error response model."""
|
"""Standard error response model."""
|
||||||
error: str = Field(..., description="Error type")
|
error: str = Field(..., description="Error type")
|
||||||
@@ -234,6 +241,90 @@ async def generate_script(request: ScriptRequest):
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@app.post("/api/tts/preview")
|
||||||
|
async def preview_voice(request: VoicePreviewRequest):
|
||||||
|
"""
|
||||||
|
Generates a short audio preview of a voice.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
request: VoicePreviewRequest containing voice ID, language, speed, and optional text
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
StreamingResponse with audio/wav content
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
400: If validation fails
|
||||||
|
500: If audio generation fails
|
||||||
|
"""
|
||||||
|
logger.info(f"Voice preview requested: voice={request.voice}, lang={request.lang}, speed={request.speed}")
|
||||||
|
|
||||||
|
# Validate language code
|
||||||
|
supported_langs = ['a', 'b', 'e', 'f', 'h', 'i', 'j', 'p', 'z']
|
||||||
|
if request.lang not in supported_langs:
|
||||||
|
logger.error(f"Invalid language code: {request.lang}")
|
||||||
|
return JSONResponse(
|
||||||
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
|
content={
|
||||||
|
"error": "ValidationError",
|
||||||
|
"message": f"Invalid language code: {request.lang}",
|
||||||
|
"details": {"field": "lang", "supported": supported_langs}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
# Validate speed range
|
||||||
|
if request.speed < 0.5 or request.speed > 1.5:
|
||||||
|
logger.error(f"Invalid speed value: {request.speed}")
|
||||||
|
return JSONResponse(
|
||||||
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
|
content={
|
||||||
|
"error": "ValidationError",
|
||||||
|
"message": "Speed must be between 0.5 and 1.5",
|
||||||
|
"details": {"field": "speed", "value": request.speed, "range": [0.5, 1.5]}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
# Use provided text or default preview text
|
||||||
|
preview_text = request.text or "Hello! This is a preview of this voice. How does it sound?"
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Get pipeline for language
|
||||||
|
pipeline = tts_service.get_pipeline(request.lang)
|
||||||
|
|
||||||
|
# Generate preview audio
|
||||||
|
audio = tts_service.synthesize_segment(
|
||||||
|
pipeline=pipeline,
|
||||||
|
text=preview_text,
|
||||||
|
voice=request.voice,
|
||||||
|
speed=request.speed
|
||||||
|
)
|
||||||
|
|
||||||
|
# Convert to WAV bytes
|
||||||
|
import io
|
||||||
|
import soundfile as sf
|
||||||
|
buffer = io.BytesIO()
|
||||||
|
sf.write(buffer, audio, tts_service.SAMPLE_RATE, format='WAV')
|
||||||
|
buffer.seek(0)
|
||||||
|
audio_bytes = buffer.read()
|
||||||
|
|
||||||
|
logger.info(f"Generated voice preview: {len(audio_bytes)} bytes")
|
||||||
|
|
||||||
|
return StreamingResponse(
|
||||||
|
iter([audio_bytes]),
|
||||||
|
media_type="audio/wav",
|
||||||
|
headers={"Content-Disposition": "inline; filename=preview.wav"}
|
||||||
|
)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error during voice preview: {str(e)}", exc_info=True)
|
||||||
|
return JSONResponse(
|
||||||
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||||
|
content={
|
||||||
|
"error": "PreviewGenerationError",
|
||||||
|
"message": "Failed to generate voice preview",
|
||||||
|
"details": {"error": str(e)}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
@app.post("/api/tts")
|
@app.post("/api/tts")
|
||||||
async def generate_audiobook(request: TTSRequest):
|
async def generate_audiobook(request: TTSRequest):
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -7,12 +7,15 @@ import AudioPlayerSection from './components/AudioPlayerSection'
|
|||||||
import type { Role, ScriptSegment } from './api/tts'
|
import type { Role, ScriptSegment } from './api/tts'
|
||||||
import { detectRoles, generateScript, generateAudiobook } from './api/tts'
|
import { detectRoles, generateScript, generateAudiobook } from './api/tts'
|
||||||
|
|
||||||
|
type AgeCategory = 'young' | 'teen' | 'adult' | 'old';
|
||||||
|
|
||||||
function App() {
|
function App() {
|
||||||
const [text, setText] = useState('')
|
const [text, setText] = useState('')
|
||||||
const [phase, setPhase] = useState<'input' | 'assignment' | 'review' | 'generated'>('input')
|
const [phase, setPhase] = useState<'input' | 'assignment' | 'review' | 'generated'>('input')
|
||||||
const [isLoading, setIsLoading] = useState(false)
|
const [isLoading, setIsLoading] = useState(false)
|
||||||
const [roles, setRoles] = useState<Role[]>([])
|
const [roles, setRoles] = useState<Role[]>([])
|
||||||
const [voiceAssignments, setVoiceAssignments] = useState<Map<string, string>>(new Map())
|
const [voiceAssignments, setVoiceAssignments] = useState<Map<string, string>>(new Map())
|
||||||
|
const [ageSettings, setAgeSettings] = useState<Map<string, AgeCategory>>(new Map())
|
||||||
const [scriptSegments, setScriptSegments] = useState<ScriptSegment[]>([])
|
const [scriptSegments, setScriptSegments] = useState<ScriptSegment[]>([])
|
||||||
const [language, setLanguage] = useState('a') // Default: American English
|
const [language, setLanguage] = useState('a') // Default: American English
|
||||||
const [speed, setSpeed] = useState(1.0) // Default: 1.0x
|
const [speed, setSpeed] = useState(1.0) // Default: 1.0x
|
||||||
@@ -56,6 +59,14 @@ function App() {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const handleAgeChange = (roleId: string, age: AgeCategory) => {
|
||||||
|
setAgeSettings(prev => {
|
||||||
|
const newMap = new Map(prev)
|
||||||
|
newMap.set(roleId, age)
|
||||||
|
return newMap
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
const handleLanguageChange = (lang: string) => {
|
const handleLanguageChange = (lang: string) => {
|
||||||
setLanguage(lang)
|
setLanguage(lang)
|
||||||
}
|
}
|
||||||
@@ -281,7 +292,9 @@ function App() {
|
|||||||
<VoiceAssignmentSection
|
<VoiceAssignmentSection
|
||||||
roles={roles}
|
roles={roles}
|
||||||
voiceAssignments={voiceAssignments}
|
voiceAssignments={voiceAssignments}
|
||||||
|
ageSettings={ageSettings}
|
||||||
onVoiceChange={handleVoiceChange}
|
onVoiceChange={handleVoiceChange}
|
||||||
|
onAgeChange={handleAgeChange}
|
||||||
language={language}
|
language={language}
|
||||||
onLanguageChange={handleLanguageChange}
|
onLanguageChange={handleLanguageChange}
|
||||||
speed={speed}
|
speed={speed}
|
||||||
|
|||||||
@@ -255,3 +255,35 @@ export async function generateAudiobook(payload: GenerateRequest): Promise<Blob>
|
|||||||
// Convert response to blob for audio data
|
// Convert response to blob for audio data
|
||||||
return await response.blob();
|
return await response.blob();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Request payload for voice preview
|
||||||
|
*/
|
||||||
|
export interface VoicePreviewRequest {
|
||||||
|
voice: string;
|
||||||
|
lang: string;
|
||||||
|
speed?: number;
|
||||||
|
text?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generate a preview of a voice
|
||||||
|
* @param payload - Preview request with voice, language, and optional speed/text
|
||||||
|
* @returns Promise resolving to audio blob
|
||||||
|
* @throws ApiError if the request fails
|
||||||
|
*/
|
||||||
|
export async function previewVoice(payload: VoicePreviewRequest): Promise<Blob> {
|
||||||
|
const baseUrl = getApiBaseUrl();
|
||||||
|
const url = `${baseUrl}/api/tts/preview`;
|
||||||
|
|
||||||
|
const response = await fetchWithErrorHandling(url, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
body: JSON.stringify(payload),
|
||||||
|
});
|
||||||
|
|
||||||
|
// Convert response to blob for audio data
|
||||||
|
return await response.blob();
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,9 +1,15 @@
|
|||||||
|
import { useState } from 'react';
|
||||||
import type { Role } from '../api/tts';
|
import type { Role } from '../api/tts';
|
||||||
|
import { previewVoice } from '../api/tts';
|
||||||
|
|
||||||
|
type AgeCategory = 'young' | 'teen' | 'adult' | 'old';
|
||||||
|
|
||||||
interface VoiceAssignmentSectionProps {
|
interface VoiceAssignmentSectionProps {
|
||||||
roles: Role[];
|
roles: Role[];
|
||||||
voiceAssignments: Map<string, string>;
|
voiceAssignments: Map<string, string>;
|
||||||
|
ageSettings: Map<string, AgeCategory>;
|
||||||
onVoiceChange: (roleId: string, voiceId: string) => void;
|
onVoiceChange: (roleId: string, voiceId: string) => void;
|
||||||
|
onAgeChange: (roleId: string, age: AgeCategory) => void;
|
||||||
language: string;
|
language: string;
|
||||||
onLanguageChange: (lang: string) => void;
|
onLanguageChange: (lang: string) => void;
|
||||||
speed: number;
|
speed: number;
|
||||||
@@ -130,10 +136,20 @@ const LANGUAGES: LanguageOption[] = [
|
|||||||
{ code: 'z', label: 'Chinese (中文)' },
|
{ code: 'z', label: 'Chinese (中文)' },
|
||||||
];
|
];
|
||||||
|
|
||||||
|
// Age category to speed mapping
|
||||||
|
const AGE_SPEED_MAP: Record<AgeCategory, number> = {
|
||||||
|
young: 1.15,
|
||||||
|
teen: 1.08,
|
||||||
|
adult: 1.0,
|
||||||
|
old: 0.85,
|
||||||
|
};
|
||||||
|
|
||||||
export default function VoiceAssignmentSection({
|
export default function VoiceAssignmentSection({
|
||||||
roles,
|
roles,
|
||||||
voiceAssignments,
|
voiceAssignments,
|
||||||
|
ageSettings,
|
||||||
onVoiceChange,
|
onVoiceChange,
|
||||||
|
onAgeChange,
|
||||||
language,
|
language,
|
||||||
onLanguageChange,
|
onLanguageChange,
|
||||||
speed,
|
speed,
|
||||||
@@ -143,8 +159,71 @@ export default function VoiceAssignmentSection({
|
|||||||
isGenerating,
|
isGenerating,
|
||||||
allRolesAssigned,
|
allRolesAssigned,
|
||||||
}: VoiceAssignmentSectionProps) {
|
}: VoiceAssignmentSectionProps) {
|
||||||
|
const [previewingRole, setPreviewingRole] = useState<string | null>(null);
|
||||||
|
const [audioCache, setAudioCache] = useState<Map<string, string>>(new Map());
|
||||||
|
|
||||||
const handleVoiceSelect = (roleId: string, voiceId: string) => {
|
const handleVoiceSelect = (roleId: string, voiceId: string) => {
|
||||||
onVoiceChange(roleId, voiceId);
|
onVoiceChange(roleId, voiceId);
|
||||||
|
// Clear cached preview when voice changes
|
||||||
|
const cacheKey = `${roleId}-${voiceId}`;
|
||||||
|
setAudioCache(prev => {
|
||||||
|
const newCache = new Map(prev);
|
||||||
|
newCache.delete(cacheKey);
|
||||||
|
return newCache;
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleAgeSelect = (roleId: string, age: AgeCategory) => {
|
||||||
|
onAgeChange(roleId, age);
|
||||||
|
// Clear cached preview when age changes
|
||||||
|
const voiceId = voiceAssignments.get(roleId);
|
||||||
|
if (voiceId) {
|
||||||
|
const cacheKey = `${roleId}-${voiceId}`;
|
||||||
|
setAudioCache(prev => {
|
||||||
|
const newCache = new Map(prev);
|
||||||
|
newCache.delete(cacheKey);
|
||||||
|
return newCache;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const handlePreview = async (roleId: string) => {
|
||||||
|
const voiceId = voiceAssignments.get(roleId);
|
||||||
|
if (!voiceId) return;
|
||||||
|
|
||||||
|
const age = ageSettings.get(roleId) || 'adult';
|
||||||
|
const previewSpeed = AGE_SPEED_MAP[age];
|
||||||
|
const cacheKey = `${roleId}-${voiceId}-${age}`;
|
||||||
|
|
||||||
|
setPreviewingRole(roleId);
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Check cache first
|
||||||
|
let audioUrl = audioCache.get(cacheKey);
|
||||||
|
|
||||||
|
if (!audioUrl) {
|
||||||
|
// Generate new preview
|
||||||
|
const audioBlob = await previewVoice({
|
||||||
|
voice: voiceId,
|
||||||
|
lang: language,
|
||||||
|
speed: previewSpeed,
|
||||||
|
});
|
||||||
|
|
||||||
|
audioUrl = URL.createObjectURL(audioBlob);
|
||||||
|
setAudioCache(prev => new Map(prev).set(cacheKey, audioUrl!));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Play the audio
|
||||||
|
if (audioUrl) {
|
||||||
|
const audio = new Audio(audioUrl);
|
||||||
|
audio.onended = () => setPreviewingRole(null);
|
||||||
|
audio.onerror = () => setPreviewingRole(null);
|
||||||
|
await audio.play();
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Preview failed:', error);
|
||||||
|
setPreviewingRole(null);
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const handleLanguageSelect = (e: React.ChangeEvent<HTMLSelectElement>) => {
|
const handleLanguageSelect = (e: React.ChangeEvent<HTMLSelectElement>) => {
|
||||||
@@ -168,30 +247,91 @@ export default function VoiceAssignmentSection({
|
|||||||
|
|
||||||
{/* Voice Assignments */}
|
{/* Voice Assignments */}
|
||||||
<div className="space-y-5">
|
<div className="space-y-5">
|
||||||
{roles.map((role, index) => (
|
{roles.map((role, index) => {
|
||||||
<div key={role.id} className="flex flex-col sm:flex-row sm:items-center gap-4 p-6 bg-gradient-to-br from-gray-50 to-white rounded-2xl border-2 border-gray-200 hover:border-indigo-300 transition-all duration-200">
|
const selectedAge = ageSettings.get(role.id) || 'adult';
|
||||||
<div className="flex items-center gap-4 sm:w-56">
|
const hasVoice = voiceAssignments.has(role.id);
|
||||||
<div className="flex items-center justify-center w-10 h-10 bg-gradient-to-br from-indigo-500 to-purple-600 text-white rounded-xl font-bold text-sm shadow-lg shadow-indigo-500/30">
|
const isPreviewing = previewingRole === role.id;
|
||||||
{index + 1}
|
|
||||||
|
return (
|
||||||
|
<div key={role.id} className="p-6 bg-gradient-to-br from-gray-50 to-white rounded-2xl border-2 border-gray-200 hover:border-indigo-300 transition-all duration-200">
|
||||||
|
{/* Header Row */}
|
||||||
|
<div className="flex flex-col sm:flex-row sm:items-center gap-4 mb-4">
|
||||||
|
<div className="flex items-center gap-4 sm:w-56">
|
||||||
|
<div className="flex items-center justify-center w-10 h-10 bg-gradient-to-br from-indigo-500 to-purple-600 text-white rounded-xl font-bold text-sm shadow-lg shadow-indigo-500/30">
|
||||||
|
{index + 1}
|
||||||
|
</div>
|
||||||
|
<label className="font-bold text-gray-900 text-lg">
|
||||||
|
{role.label}
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
<select
|
||||||
|
value={voiceAssignments.get(role.id) || ''}
|
||||||
|
onChange={(e) => handleVoiceSelect(role.id, e.target.value)}
|
||||||
|
className="flex-1 p-4 border-2 border-gray-200 rounded-xl focus:border-indigo-500 focus:ring-4 focus:ring-indigo-100 focus:outline-none bg-white text-base font-medium transition-all duration-200"
|
||||||
|
>
|
||||||
|
<option value="">Select a voice...</option>
|
||||||
|
{KOKORO_VOICES.map((voice) => (
|
||||||
|
<option key={voice.id} value={voice.id}>
|
||||||
|
{voice.label}
|
||||||
|
</option>
|
||||||
|
))}
|
||||||
|
</select>
|
||||||
</div>
|
</div>
|
||||||
<label className="font-bold text-gray-900 text-lg">
|
|
||||||
{role.label}
|
{/* Age Category & Preview Row */}
|
||||||
</label>
|
{hasVoice && (
|
||||||
|
<div className="flex flex-col sm:flex-row gap-4 items-start sm:items-center">
|
||||||
|
{/* Age Categories */}
|
||||||
|
<div className="flex-1">
|
||||||
|
<label className="block text-sm font-semibold text-gray-700 mb-2">
|
||||||
|
Age Category:
|
||||||
|
</label>
|
||||||
|
<div className="flex gap-2 flex-wrap">
|
||||||
|
{(['young', 'teen', 'adult', 'old'] as AgeCategory[]).map((age) => (
|
||||||
|
<button
|
||||||
|
key={age}
|
||||||
|
onClick={() => handleAgeSelect(role.id, age)}
|
||||||
|
className={`px-4 py-2 rounded-lg font-semibold text-sm transition-all duration-200 ${
|
||||||
|
selectedAge === age
|
||||||
|
? 'bg-indigo-600 text-white shadow-lg shadow-indigo-500/30'
|
||||||
|
: 'bg-white text-gray-700 border-2 border-gray-200 hover:border-indigo-300'
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
{age === 'young' && '👶 Young'}
|
||||||
|
{age === 'teen' && '🧒 Teen'}
|
||||||
|
{age === 'adult' && '👤 Adult'}
|
||||||
|
{age === 'old' && '👴 Old'}
|
||||||
|
</button>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
<p className="text-xs text-gray-500 mt-2">
|
||||||
|
Speed: {AGE_SPEED_MAP[selectedAge]}x
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Preview Button */}
|
||||||
|
<button
|
||||||
|
onClick={() => handlePreview(role.id)}
|
||||||
|
disabled={isPreviewing || isGenerating}
|
||||||
|
className="px-6 py-3 bg-green-600 text-white font-semibold rounded-xl hover:bg-green-700 disabled:opacity-50 disabled:cursor-not-allowed transition-all duration-200 flex items-center gap-2 shadow-lg shadow-green-500/30"
|
||||||
|
>
|
||||||
|
{isPreviewing ? (
|
||||||
|
<>
|
||||||
|
<div className="animate-spin rounded-full h-4 w-4 border-2 border-white border-t-transparent"></div>
|
||||||
|
Playing...
|
||||||
|
</>
|
||||||
|
) : (
|
||||||
|
<>
|
||||||
|
<span>🔊</span>
|
||||||
|
Try Voice
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
<select
|
);
|
||||||
value={voiceAssignments.get(role.id) || ''}
|
})}
|
||||||
onChange={(e) => handleVoiceSelect(role.id, e.target.value)}
|
|
||||||
className="flex-1 p-4 border-2 border-gray-200 rounded-xl focus:border-indigo-500 focus:ring-4 focus:ring-indigo-100 focus:outline-none bg-white text-base font-medium transition-all duration-200"
|
|
||||||
>
|
|
||||||
<option value="">Select a voice...</option>
|
|
||||||
{KOKORO_VOICES.map((voice) => (
|
|
||||||
<option key={voice.id} value={voice.id}>
|
|
||||||
{voice.label}
|
|
||||||
</option>
|
|
||||||
))}
|
|
||||||
</select>
|
|
||||||
</div>
|
|
||||||
))}
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* Settings Grid */}
|
{/* Settings Grid */}
|
||||||
|
|||||||
Reference in New Issue
Block a user