Old Tam rewritten and re-voiced (Gitea #3): a new script in his own voice, an old West Country farmer (Qwen3-TTS cloning one recording, so he's the same man in every line), each line's best take chosen by checking its words with Whisper
This commit is contained in:
Paul Bailey
@@ -1,23 +1,33 @@
|
||||
"""Old Tam's voice: every line he says (the tutorial's steps and goodbye in
|
||||
data/tutorial.json, his tips in data/hints.json), spoken by Kokoro (Kokoro-82M,
|
||||
Apache-2.0, hexgrad) with its British "George" voice, then made a little older:
|
||||
deeper, slower and warmer. Output: assets/audio/voice/<line id>.ogg plus
|
||||
lines.json, the text each file was made from (the game only plays a file
|
||||
whose text still matches, so an edited line goes quiet rather than saying
|
||||
the old words; tests/test_tutorial.gd checks they're all up to date).
|
||||
data/tutorial.json, his tips in data/hints.json), in his own voice: Qwen3-TTS
|
||||
(Alibaba's Qwen team, Apache-2.0) cloning tools/voice/old_tam/reference.wav,
|
||||
a reedy old West Country farmer that Qwen3's VoiceDesign made from a written
|
||||
description (old_tam/voice.json). Cloning that one recording, rather than
|
||||
describing him again for each line, keeps him the same man throughout.
|
||||
|
||||
Run it with tools/voice/make_voice.sh (sets up the Kokoro environment once).
|
||||
Only lines whose text changed are re-made; --all re-makes everything.
|
||||
Each line is recorded TAKES times; Whisper (mlx-whisper) writes down what
|
||||
each take says, and the take whose words best match the line is kept (a take
|
||||
that drops or garbles words loses). Output: assets/audio/voice/<line id>.ogg,
|
||||
lines.json (the written text each file was made from: the game only plays a
|
||||
file whose text still matches, so an edited line goes quiet rather than
|
||||
saying the old words; tests/test_tutorial.gd checks they're all up to date)
|
||||
and spoken.json (what each one says).
|
||||
|
||||
Run it with tools/voice/make_voice.sh (sets up the environment once, on an
|
||||
Apple-silicon Mac: MLX). Only lines whose text, or how they're spoken, changed
|
||||
are re-made; --all re-makes everything; --takes N for more takes a line.
|
||||
"""
|
||||
import hashlib, json, os, re, subprocess, sys, tempfile
|
||||
import difflib, hashlib, json, os, re, subprocess, sys, tempfile
|
||||
|
||||
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
||||
OUT = os.path.join(ROOT, "assets", "audio", "voice")
|
||||
VOICE = "bm_george"
|
||||
SPEED = 0.9
|
||||
# Older: down about 1.4 semitones (and so a touch slower), then a little
|
||||
# warmth and the top taken off.
|
||||
AGE = "asetrate=24000*0.92,aresample=24000,atempo=1.04,lowpass=f=6500,bass=g=2:f=180"
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
VOICE = json.load(open(os.path.join(HERE, "old_tam", "voice.json")))
|
||||
MODEL = VOICE["model"]
|
||||
REF_AUDIO = os.path.join(HERE, "old_tam", VOICE["reference_audio"])
|
||||
REF_TEXT = VOICE["reference_text"]
|
||||
WHISPER = "mlx-community/whisper-large-v3-turbo"
|
||||
TAKES = 3
|
||||
|
||||
|
||||
def lines():
|
||||
@@ -45,19 +55,24 @@ def spoken(text):
|
||||
s = s.replace(" · ", ", ").replace("&", "and")
|
||||
s = re.sub(r"\bLMB\b", "left click", s)
|
||||
s = re.sub(r"\bRMB\b", "right click", s)
|
||||
s = re.sub(r"\bWASD\b", "W, A, S and D", s) # (Gitea #3: it said "Wasad")
|
||||
# (Gitea #3: spelt out, "W, A, S and D" comes out rushed; the screen shows the keys.)
|
||||
s = re.sub(r"\bWASD\b", "the keys", s)
|
||||
s = re.sub(r"\s+,", ",", s)
|
||||
return s
|
||||
|
||||
|
||||
def _changed_by_new_rules(text):
|
||||
"""Whether a line has something the spoken() rules added since the last
|
||||
full recording (0.6.2): "WASD"."""
|
||||
return re.search(r"\bWASD\b", text) is not None
|
||||
def _words(s):
|
||||
return re.sub(r"[^a-z' ]", " ", s.lower().replace("-", " ")).split()
|
||||
|
||||
|
||||
def match(want, heard):
|
||||
"""How closely what Whisper heard matches the line (0..1)."""
|
||||
return difflib.SequenceMatcher(None, _words(want), _words(heard)).ratio()
|
||||
|
||||
|
||||
def main():
|
||||
redo_all = "--all" in sys.argv
|
||||
takes = int(sys.argv[sys.argv.index("--takes") + 1]) if "--takes" in sys.argv else TAKES
|
||||
os.makedirs(OUT, exist_ok=True)
|
||||
manifest_path = os.path.join(OUT, "lines.json")
|
||||
old = json.load(open(manifest_path)) if os.path.exists(manifest_path) else {}
|
||||
@@ -66,38 +81,57 @@ def main():
|
||||
# key: that's what the game checks against the screen.)
|
||||
said_path = os.path.join(OUT, "spoken.json")
|
||||
said = json.load(open(said_path)) if os.path.exists(said_path) else {}
|
||||
voice_key = key(json.dumps(VOICE, sort_keys=True))
|
||||
new_voice = said.get("_voice") != voice_key
|
||||
todo = [(i, t) for i, t in lines()
|
||||
if redo_all or old.get(i) != key(t) or said.get(i, key(spoken(t))) != key(spoken(t))
|
||||
if redo_all or new_voice or old.get(i) != key(t) or said.get(i) != key(spoken(t))
|
||||
or not os.path.exists(os.path.join(OUT, i + ".ogg"))]
|
||||
if not said:
|
||||
# (First run with spoken.json: lines spoken differently now than when
|
||||
# they were recorded can't be told apart, so re-record the ones whose
|
||||
# spoken form isn't just their text.)
|
||||
todo += [(i, t) for i, t in lines() if spoken(t) != t and (i, t) not in todo and _changed_by_new_rules(t)]
|
||||
keep = {i for i, _ in lines()}
|
||||
for f in os.listdir(OUT): # lines that no longer exist
|
||||
if f.endswith(".ogg") and f[:-4] not in keep:
|
||||
os.remove(os.path.join(OUT, f))
|
||||
report = []
|
||||
if todo:
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
from kokoro import KPipeline
|
||||
pipe = KPipeline(lang_code="b", repo_id="hexgrad/Kokoro-82M")
|
||||
for i, text in todo:
|
||||
audio = np.concatenate([a for _, _, a in pipe(spoken(text), voice=VOICE, speed=SPEED)])
|
||||
tmp = tempfile.NamedTemporaryFile(suffix=".wav", delete=False).name
|
||||
sf.write(tmp, audio, 24000)
|
||||
import mlx_whisper
|
||||
from mlx_audio.tts.utils import load_model
|
||||
model = load_model(MODEL)
|
||||
sr = getattr(model, "sample_rate", 24000)
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
for n, (i, text) in enumerate(todo, 1):
|
||||
say = spoken(text)
|
||||
best = None
|
||||
for k in range(takes):
|
||||
res = list(model.generate(text=say, ref_audio=REF_AUDIO, ref_text=REF_TEXT))
|
||||
audio = np.concatenate([np.array(r.audio) for r in res])
|
||||
wav = os.path.join(tmpdir, "%s_%d.wav" % (i, k))
|
||||
sf.write(wav, audio, sr)
|
||||
heard = mlx_whisper.transcribe(wav, path_or_hf_repo=WHISPER, language="en")["text"].strip()
|
||||
score = match(say, heard)
|
||||
if best is None or score > best[0]:
|
||||
best = (score, wav, heard, len(audio) / sr)
|
||||
if score >= 0.98:
|
||||
break # (word for word: no need for another)
|
||||
score, wav, heard, secs = best
|
||||
path = os.path.join(OUT, i + ".ogg")
|
||||
# (ffmpeg's own Vorbis encoder only writes stereo)
|
||||
subprocess.run(["ffmpeg", "-v", "error", "-y", "-i", tmp, "-af", AGE + ",apad=pad_dur=0.25", "-ac", "2",
|
||||
"-c:a", "vorbis", "-strict", "-2", "-q:a", "4", path], check=True)
|
||||
os.remove(tmp)
|
||||
subprocess.run(["ffmpeg", "-v", "error", "-y", "-i", wav, "-af", "apad=pad_dur=0.25", "-ac", "2",
|
||||
"-c:a", "vorbis", "-strict", "-2", "-q:a", "5", path], check=True)
|
||||
old[i] = key(text)
|
||||
print(" %s (%.1f s)" % (i, len(audio) / 24000 / 0.92 / 1.04))
|
||||
said[i] = key(say)
|
||||
flag = "" if score >= 0.9 else " <-- check this one: heard \"%s\"" % heard
|
||||
report.append((score, i, heard))
|
||||
print(" [%d/%d] %s (%.1f s, words %.0f%%)%s" % (n, len(todo), i, secs, score * 100, flag))
|
||||
manifest = {i: key(t) for i, t in lines()}
|
||||
json.dump(manifest, open(manifest_path, "w"), indent="\t", sort_keys=True)
|
||||
json.dump({i: key(spoken(t)) for i, t in lines()}, open(said_path, "w"), indent="\t", sort_keys=True)
|
||||
out_said = {i: said.get(i, key(spoken(t))) for i, t in lines()}
|
||||
out_said["_voice"] = voice_key
|
||||
json.dump(out_said, open(said_path, "w"), indent="\t", sort_keys=True)
|
||||
print("%d lines, %d made now -> %s" % (len(manifest), len(todo), os.path.relpath(OUT, ROOT)))
|
||||
weak = [r for r in report if r[0] < 0.9]
|
||||
if weak:
|
||||
print("%d to listen to (the words didn't quite match): %s" % (len(weak), ", ".join(r[1] for r in weak)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -5,17 +5,18 @@
|
||||
# tools/voice/make_voice.sh # just the lines that changed
|
||||
# tools/voice/make_voice.sh --all # every line again
|
||||
#
|
||||
# The first run sets up a Python 3.12 environment for Kokoro in
|
||||
# tools/voice/.venv (about 1 GB, with PyTorch; needs uv) and downloads the
|
||||
# Kokoro model (about 330 MB) from Hugging Face.
|
||||
# Needs an Apple-silicon Mac (MLX), ffmpeg and uv. The first run sets up a
|
||||
# Python 3.12 environment in tools/voice/.venv-qwen and downloads the models
|
||||
# from Hugging Face: Qwen3-TTS 1.7B Base (about 4 GB) and Whisper large-v3
|
||||
# turbo (about 1.6 GB), to check what each take says.
|
||||
|
||||
cd "$(dirname "$0")/../.." || exit 1
|
||||
V=tools/voice/.venv
|
||||
V=tools/voice/.venv-qwen
|
||||
command -v ffmpeg >/dev/null || { echo "needs ffmpeg (brew install ffmpeg)"; exit 1; }
|
||||
if [ ! -x "$V/bin/python" ]; then
|
||||
command -v uv >/dev/null || { echo "needs uv (https://docs.astral.sh/uv/)"; exit 1; }
|
||||
uv venv -q --python 3.12 "$V" || exit 1
|
||||
uv pip install -q --python "$V/bin/python" "kokoro==0.9.4" "misaki[en]==0.9.4" "soundfile==0.14.0" \
|
||||
"en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl" || exit 1
|
||||
uv pip install -q --python "$V/bin/python" "mlx-audio==0.6.4" "mlx==0.32.4" "mlx-whisper==0.4.3" \
|
||||
"soundfile==0.13.1" "numpy==2.5.4" "transformers==5.22.2" || exit 1
|
||||
fi
|
||||
exec "$V/bin/python" tools/voice/make_voice.py "$@"
|
||||
|
||||
BIN
tools/voice/old_tam/reference.wav
Normal file
BIN
tools/voice/old_tam/reference.wav
Normal file
Binary file not shown.
7
tools/voice/old_tam/voice.json
Normal file
7
tools/voice/old_tam/voice.json
Normal file
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"_doc": "Old Tam's voice (tools/voice/make_voice.py). reference.wav was made once by Qwen3-TTS VoiceDesign (mlx-community/Qwen3-TTS-12Hz-1.7B-VoiceDesign-bf16) from `description`, saying `reference_text`; Paul picked it (Nov 2026). Every line is then spoken by the Base model cloning that recording, so it's the same voice throughout. Change any of this and every line is re-made.",
|
||||
"description": "An elderly, toothless-sounding old man with a thin, reedy, quavering voice and a broad West Country English accent, like an eccentric old farmer. Chuckling, rambling and full of mischief, a little out of breath.",
|
||||
"model": "mlx-community/Qwen3-TTS-12Hz-1.7B-Base-bf16",
|
||||
"reference_audio": "reference.wav",
|
||||
"reference_text": "Ha! Awake at last! Washed up on my island, eh? Well, stretch your legs: W, A, S and D to walk, the mouse to look. Shift if you're in a hurry."
|
||||
}
|
||||
Reference in New Issue
Block a user