80 lines
3.7 KiB
Python
80 lines
3.7 KiB
Python
"""Old Tam narrates the trailer (tools/trailer/make_trailer.sh mixes it in):
|
|
one line for each shot, in his voice (the same clone as
|
|
tools/voice/make_voice.py), each checked by Whisper, the best of three takes
|
|
kept, and sped up a touch if it would run past the start of the next line.
|
|
Output: tools/trailer/voice/<NN>.ogg and lines.json ([{file, shot, at, text}]:
|
|
`at` is seconds into its shot; make_trailer.sh reads the shots' start times
|
|
from the render log). Run with the voice environment:
|
|
tools/voice/.venv-qwen/bin/python tools/trailer/make_trailer_voice.py
|
|
"""
|
|
import json, os, subprocess, sys, tempfile
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
sys.path.insert(0, os.path.join(HERE, "..", "voice"))
|
|
import make_voice as mv # (its voice, its checks)
|
|
|
|
OUT = os.path.join(HERE, "voice")
|
|
## The shots (trailer.gd), their lengths in seconds, and what he says over
|
|
## each, starting `at` seconds in.
|
|
SHOTS = [7.5, 5.0, 6.0, 5.5, 4.5, 5.5, 6.0, 6.0]
|
|
LINES = [
|
|
(0, 2.0, "Welcome to Cinderwild. You wake on a wild island, washed up with nothing but the clothes on your back."),
|
|
(1, 0.3, "Gather what you can. Hunt, forage, and cook over an open fire."),
|
|
(2, 0.3, "Build yourself a home, plank by plank, before the nights draw in."),
|
|
(3, 0.3, "Because when the sun goes down, the wolves come out. Keep your fire burning."),
|
|
(4, 0.3, "Weather the storms, and the hard winters."),
|
|
(5, 0.3, "Then dig down, through the sea cliffs."),
|
|
(6, 0.3, "And find out what lies beyond."),
|
|
(7, 0.6, "Play alone, or with your friends, on your own server. Cinderwild."),
|
|
]
|
|
GAP = 0.25 # seconds of quiet before the next line
|
|
|
|
|
|
def main():
|
|
import numpy as np
|
|
import soundfile as sf
|
|
import mlx_whisper
|
|
from mlx_audio.tts.utils import load_model
|
|
os.makedirs(OUT, exist_ok=True)
|
|
for f in os.listdir(OUT):
|
|
if f.endswith(".ogg"):
|
|
os.remove(os.path.join(OUT, f))
|
|
model = load_model(mv.MODEL)
|
|
sr = getattr(model, "sample_rate", 24000)
|
|
tmp = tempfile.mkdtemp()
|
|
out = []
|
|
for n, (shot, at, text) in enumerate(LINES):
|
|
# Room: to the next line's start (or the end of the film).
|
|
if n + 1 < len(LINES):
|
|
nshot, nat, _ = LINES[n + 1]
|
|
room = sum(SHOTS[shot:nshot]) - at + nat - GAP
|
|
else:
|
|
room = sum(SHOTS[shot:]) - at - 0.5
|
|
best = None
|
|
for k in range(3):
|
|
res = list(model.generate(text=text, ref_audio=mv.REF_AUDIO, ref_text=mv.REF_TEXT))
|
|
audio = np.concatenate([np.array(r.audio) for r in res])
|
|
wav = os.path.join(tmp, "%02d_%d.wav" % (n, k))
|
|
sf.write(wav, audio, sr)
|
|
heard = mlx_whisper.transcribe(wav, path_or_hf_repo=mv.WHISPER, language="en")["text"].strip()
|
|
secs = len(audio) / sr
|
|
# Words first; then the one that fits.
|
|
score = mv.match(text, heard) - (0.05 if secs > room * 1.15 else 0.0)
|
|
if best is None or score > best[0]:
|
|
best = (score, wav, secs)
|
|
if score >= 0.98 and secs <= room:
|
|
break
|
|
score, wav, secs = best
|
|
tempo = min(max(secs / room, 1.0), 1.2) if room > 0 else 1.0
|
|
name = "%02d.ogg" % n
|
|
subprocess.run(["ffmpeg", "-v", "error", "-y", "-i", wav, "-af", "atempo=%.3f" % tempo, "-ac", "2",
|
|
"-c:a", "vorbis", "-strict", "-2", "-q:a", "5", os.path.join(OUT, name)], check=True)
|
|
out.append({"file": name, "shot": shot, "at": at, "text": text})
|
|
print(" %s shot %d: %.1f s%s (room %.1f s, words %.0f%%)" % (name, shot, secs / tempo,
|
|
" (x%.2f)" % tempo if tempo > 1.0 else "", room, score * 100), flush=True)
|
|
json.dump(out, open(os.path.join(OUT, "lines.json"), "w"), indent=2)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|