139 lines
6.4 KiB
Python
139 lines
6.4 KiB
Python
"""Old Tam's voice: every line he says (the tutorial's steps and goodbye in
|
|
data/tutorial.json, his tips in data/hints.json), in his own voice: Qwen3-TTS
|
|
(Alibaba's Qwen team, Apache-2.0) cloning tools/voice/old_tam/reference.wav,
|
|
a reedy old West Country farmer that Qwen3's VoiceDesign made from a written
|
|
description (old_tam/voice.json). Cloning that one recording, rather than
|
|
describing him again for each line, keeps him the same man throughout.
|
|
|
|
Each line is recorded TAKES times; Whisper (mlx-whisper) writes down what
|
|
each take says, and the take whose words best match the line is kept (a take
|
|
that drops or garbles words loses). Output: assets/audio/voice/<line id>.ogg,
|
|
lines.json (the written text each file was made from: the game only plays a
|
|
file whose text still matches, so an edited line goes quiet rather than
|
|
saying the old words; tests/test_tutorial.gd checks they're all up to date)
|
|
and spoken.json (what each one says).
|
|
|
|
Run it with tools/voice/make_voice.sh (sets up the environment once, on an
|
|
Apple-silicon Mac: MLX). Only lines whose text, or how they're spoken, changed
|
|
are re-made; --all re-makes everything; --takes N for more takes a line.
|
|
"""
|
|
import difflib, hashlib, json, os, re, subprocess, sys, tempfile
|
|
|
|
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
|
OUT = os.path.join(ROOT, "assets", "audio", "voice")
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
VOICE = json.load(open(os.path.join(HERE, "old_tam", "voice.json")))
|
|
MODEL = VOICE["model"]
|
|
REF_AUDIO = os.path.join(HERE, "old_tam", VOICE["reference_audio"])
|
|
REF_TEXT = VOICE["reference_text"]
|
|
WHISPER = "mlx-community/whisper-large-v3-turbo"
|
|
TAKES = 3
|
|
|
|
|
|
def lines():
|
|
"""[(id, text)] for everything Old Tam says."""
|
|
out = []
|
|
t = json.load(open(os.path.join(ROOT, "data", "tutorial.json")))
|
|
for s in t["steps"]:
|
|
out.append(("tutorial_" + s["id"], s["text"]))
|
|
if t.get("finish", {}).get("text"):
|
|
out.append(("tutorial_finish", t["finish"]["text"]))
|
|
h = json.load(open(os.path.join(ROOT, "data", "hints.json")))
|
|
for x in h["hints"]:
|
|
out.append(("hint_" + x["id"], x["text"]))
|
|
return out
|
|
|
|
|
|
def key(text):
|
|
return hashlib.sha1(text.encode("utf-8")).hexdigest()[:16]
|
|
|
|
|
|
def spoken(text):
|
|
"""What to say for what's written: keys read as words, no brackets."""
|
|
s = text.replace("(E)", "with E").replace("[E]", "E")
|
|
s = re.sub(r"\(([^)]*)\)", r", \1,", s) # asides become pauses
|
|
s = s.replace(" · ", ", ").replace("&", "and")
|
|
s = re.sub(r"\bLMB\b", "left click", s)
|
|
s = re.sub(r"\bRMB\b", "right click", s)
|
|
# (Gitea #3: spelt out, "W, A, S and D" comes out rushed; the screen shows the keys.)
|
|
s = re.sub(r"\bWASD\b", "the keys", s)
|
|
s = re.sub(r"\s+,", ",", s)
|
|
return s
|
|
|
|
|
|
def _words(s):
|
|
return re.sub(r"[^a-z' ]", " ", s.lower().replace("-", " ")).split()
|
|
|
|
|
|
def match(want, heard):
|
|
"""How closely what Whisper heard matches the line (0..1)."""
|
|
return difflib.SequenceMatcher(None, _words(want), _words(heard)).ratio()
|
|
|
|
|
|
def main():
|
|
redo_all = "--all" in sys.argv
|
|
takes = int(sys.argv[sys.argv.index("--takes") + 1]) if "--takes" in sys.argv else TAKES
|
|
os.makedirs(OUT, exist_ok=True)
|
|
manifest_path = os.path.join(OUT, "lines.json")
|
|
old = json.load(open(manifest_path)) if os.path.exists(manifest_path) else {}
|
|
# What each recording SAYS (spoken() of its text), so a change to how a
|
|
# line is spoken re-records it too. (lines.json keeps the written text's
|
|
# key: that's what the game checks against the screen.)
|
|
said_path = os.path.join(OUT, "spoken.json")
|
|
said = json.load(open(said_path)) if os.path.exists(said_path) else {}
|
|
voice_key = key(json.dumps(VOICE, sort_keys=True))
|
|
new_voice = said.get("_voice") != voice_key
|
|
todo = [(i, t) for i, t in lines()
|
|
if redo_all or new_voice or old.get(i) != key(t) or said.get(i) != key(spoken(t))
|
|
or not os.path.exists(os.path.join(OUT, i + ".ogg"))]
|
|
keep = {i for i, _ in lines()}
|
|
for f in os.listdir(OUT): # lines that no longer exist
|
|
if f.endswith(".ogg") and f[:-4] not in keep:
|
|
os.remove(os.path.join(OUT, f))
|
|
report = []
|
|
if todo:
|
|
import numpy as np
|
|
import soundfile as sf
|
|
import mlx_whisper
|
|
from mlx_audio.tts.utils import load_model
|
|
model = load_model(MODEL)
|
|
sr = getattr(model, "sample_rate", 24000)
|
|
tmpdir = tempfile.mkdtemp()
|
|
for n, (i, text) in enumerate(todo, 1):
|
|
say = spoken(text)
|
|
best = None
|
|
for k in range(takes):
|
|
res = list(model.generate(text=say, ref_audio=REF_AUDIO, ref_text=REF_TEXT))
|
|
audio = np.concatenate([np.array(r.audio) for r in res])
|
|
wav = os.path.join(tmpdir, "%s_%d.wav" % (i, k))
|
|
sf.write(wav, audio, sr)
|
|
heard = mlx_whisper.transcribe(wav, path_or_hf_repo=WHISPER, language="en")["text"].strip()
|
|
score = match(say, heard)
|
|
if best is None or score > best[0]:
|
|
best = (score, wav, heard, len(audio) / sr)
|
|
if score >= 0.98:
|
|
break # (word for word: no need for another)
|
|
score, wav, heard, secs = best
|
|
path = os.path.join(OUT, i + ".ogg")
|
|
# (ffmpeg's own Vorbis encoder only writes stereo)
|
|
subprocess.run(["ffmpeg", "-v", "error", "-y", "-i", wav, "-af", "apad=pad_dur=0.25", "-ac", "2",
|
|
"-c:a", "vorbis", "-strict", "-2", "-q:a", "5", path], check=True)
|
|
old[i] = key(text)
|
|
said[i] = key(say)
|
|
flag = "" if score >= 0.9 else " <-- check this one: heard \"%s\"" % heard
|
|
report.append((score, i, heard))
|
|
print(" [%d/%d] %s (%.1f s, words %.0f%%)%s" % (n, len(todo), i, secs, score * 100, flag))
|
|
manifest = {i: key(t) for i, t in lines()}
|
|
json.dump(manifest, open(manifest_path, "w"), indent="\t", sort_keys=True)
|
|
out_said = {i: said.get(i, key(spoken(t))) for i, t in lines()}
|
|
out_said["_voice"] = voice_key
|
|
json.dump(out_said, open(said_path, "w"), indent="\t", sort_keys=True)
|
|
print("%d lines, %d made now -> %s" % (len(manifest), len(todo), os.path.relpath(OUT, ROOT)))
|
|
weak = [r for r in report if r[0] < 0.9]
|
|
if weak:
|
|
print("%d to listen to (the words didn't quite match): %s" % (len(weak), ", ".join(r[1] for r in weak)))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|