248 lines
9.7 KiB
Python
248 lines
9.7 KiB
Python
"""whisper large-v3 vs large-v2 同配置对比实验脚本（savr-1054 全片）。
|
||
|
||
目的：验证 large-v2 是否比当前生产用的 large-v3 更适合本项目（日语成人内容，
|
||
呻吟/BGM 混叠、弱语音）。方法：对同一视频，用相同参数分别跑 large-v3 与
|
||
large-v2 的「生产 VAD」与「无 VAD」两条链路，统计条数/覆盖时长/幻觉长段等
|
||
指标，与 docs/调研-whisper漏句与decode_full验证.md 中记录的 v3 历史结果对照。
|
||
|
||
复现路径：直接调用 nodes/whisper.py 的 invoke（真实节点代码），不绕开注册表
|
||
或重写业务逻辑。显式传 vad_parameters 复现文档实验一的"生产 VAD
|
||
（threshold0.5/ms1000/pad200）"，并关闭自动 VAD（WOV_AUTO_VAD=0），确保
|
||
v3/v2 差异纯粹来自模型，而非信号分析分支抖动。
|
||
|
||
用法：
|
||
uv run python scripts/compare_whisper_v2_vs_v3.py \
|
||
--video /mnt/fnOS/123/savr-1054/4k2.me@savr01054_2_8k.mp4 \
|
||
--out data/experiments/whisper_v2_vs_v3
|
||
|
||
产物：
|
||
<out>/audio.wav 16k 单声道（ffmpeg 提取一次，复用）
|
||
<out>/<tag>/transcript.srt 各组合的 SRT 产物
|
||
<out>/summary.csv 指标汇总
|
||
控制台打印对比表
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import csv
|
||
import os
|
||
import re
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
from pathlib import Path
|
||
|
||
# 保证脚本可从仓库根目录直接 import nodes/wov_sdk。
|
||
ROOT = Path(__file__).resolve().parent.parent
|
||
sys.path.insert(0, str(ROOT))
|
||
os.environ.setdefault("WOV_AUTO_VAD", "0") # 关闭自动 VAD，用显式参数
|
||
|
||
from nodes.ffmpeg import _ffmpeg_bin # noqa: E402
|
||
from nodes.subtitle_cleanup import JAPANESE_HALLUCINATION_TOKENS # noqa: E402
|
||
from wov_sdk.models import InvokeRequest # noqa: E402
|
||
|
||
# 实验一的生产 VAD 配置（文档记录）：threshold0.5 / min_silence1000 / pad200。
|
||
PROD_VAD_PARAMS = {
|
||
"threshold": 0.5,
|
||
"min_silence_duration_ms": 1000,
|
||
"speech_pad_ms": 200,
|
||
}
|
||
|
||
# 幻觉长段判定：展示时长 ≥ 阈值 且文本命中日文寒暄词表（与 subtitle_cleanup
|
||
# 的 decode_full 清洗同一判据，这里不删除只统计）。
|
||
HALLUC_THRESHOLD_SECONDS = 15.0
|
||
|
||
VIDEO_PATH = "/mnt/fnOS/123/savr-1054/4k2.me@savr01054_2_8k.mp4"
|
||
OUT_DIR = ROOT / "data" / "experiments" / "whisper_v2_vs_v3"
|
||
|
||
|
||
def extract_audio(video: Path, out_dir: Path) -> Path:
|
||
"""用 ffmpeg 提取 16k 单声道 wav（与 ffmpeg-extract 节点相同规格）。"""
|
||
wav = out_dir / "audio.wav"
|
||
if wav.is_file() and wav.stat().st_size > 0:
|
||
return wav
|
||
subprocess.run(
|
||
[
|
||
_ffmpeg_bin(),
|
||
"-y", "-i", str(video),
|
||
"-vn", "-ac", "1", "-ar", "16000",
|
||
str(wav),
|
||
],
|
||
check=True, capture_output=True,
|
||
)
|
||
return wav
|
||
|
||
|
||
def parse_srt(srt_text: str) -> list[dict]:
|
||
"""解析 SRT 文本为 cue 列表（序号/起止/文本/时长），文档统计口径。"""
|
||
cues = []
|
||
# 标准 SRT 块：序号 / 时间轴 / 文本行（可多行）/ 空行。
|
||
blocks = re.split(r"\n\s*\n", srt_text.strip())
|
||
for block in blocks:
|
||
lines = [ln for ln in block.splitlines() if ln.strip()]
|
||
if len(lines) < 2:
|
||
continue
|
||
if not lines[0].strip().isdigit():
|
||
continue
|
||
m = re.match(
|
||
r"(\d+):(\d+):(\d+)[,.](\d+)\s*-->\s*(\d+):(\d+):(\d+)[,.](\d+)",
|
||
lines[1],
|
||
)
|
||
if not m:
|
||
continue
|
||
start = (int(m[1]) * 3600 + int(m[2]) * 60 + int(m[3])) + int(m[4]) / 1000
|
||
end = (int(m[5]) * 3600 + int(m[6]) * 60 + int(m[7])) + int(m[8]) / 1000
|
||
text = " ".join(lines[2:]).strip()
|
||
cues.append({
|
||
"start": start,
|
||
"end": end,
|
||
"duration": end - start,
|
||
"text": text,
|
||
})
|
||
return cues
|
||
|
||
|
||
def stats(cues: list[dict], total_seconds: float) -> dict:
|
||
"""汇总指标：条数/展示时长和/覆盖时长(并集)/幻觉长段。"""
|
||
count = len(cues)
|
||
display_sum = sum(c["duration"] for c in cues)
|
||
# 覆盖并集：按开始时间排序后合并重叠区间。
|
||
ordered = sorted(cues, key=lambda c: c["start"])
|
||
union = 0.0
|
||
cur_s, cur_e = None, None
|
||
for c in ordered:
|
||
if cur_s is None:
|
||
cur_s, cur_e = c["start"], c["end"]
|
||
elif c["start"] <= cur_e:
|
||
cur_e = max(cur_e, c["end"])
|
||
else:
|
||
union += cur_e - cur_s
|
||
cur_s, cur_e = c["start"], c["end"]
|
||
if cur_s is not None:
|
||
union += cur_e - cur_s
|
||
# 幻觉长段：时长≥阈值 且 文本含任一寒暄词。
|
||
halluc = [
|
||
c for c in cues
|
||
if c["duration"] >= HALLUC_THRESHOLD_SECONDS
|
||
and any(tok in c["text"] for tok in JAPANESE_HALLUCINATION_TOKENS)
|
||
]
|
||
halluc_seconds = sum(c["duration"] for c in halluc)
|
||
return {
|
||
"count": count,
|
||
"display_sum": round(display_sum, 1),
|
||
"union": round(union, 1),
|
||
"coverage_pct": round(union / total_seconds * 100, 1) if total_seconds else 0.0,
|
||
"halluc_count": len(halluc),
|
||
"halluc_seconds": round(halluc_seconds, 1),
|
||
}
|
||
|
||
|
||
def run_combo(tag: str, model_path: str, wav: Path, out_dir: Path,
|
||
vad_filter: bool, vad_parameters: dict | None) -> dict:
|
||
"""跑一趟 whisper.invoke（真实节点路径），返回统计与产物路径。"""
|
||
combo_dir = out_dir / tag
|
||
combo_dir.mkdir(parents=True, exist_ok=True)
|
||
params: dict = {
|
||
"language": "ja",
|
||
"model_path": model_path,
|
||
"condition_on_previous_text": False,
|
||
"chunk_seconds": 60,
|
||
"vad_filter": vad_filter,
|
||
"beam_size": 1,
|
||
}
|
||
if vad_parameters is not None:
|
||
params["vad_parameters"] = vad_parameters
|
||
request = InvokeRequest(
|
||
run_id=f"exp-{tag}",
|
||
node_instance_id="whisper",
|
||
inputs={"audio_uri": str(wav)},
|
||
params=params,
|
||
output_dir=str(combo_dir),
|
||
)
|
||
started = time.monotonic()
|
||
from nodes.whisper import invoke
|
||
response = invoke(request)
|
||
elapsed = time.monotonic() - started
|
||
if response.status != "completed":
|
||
raise RuntimeError(f"[{tag}] invoke failed: {response.error}")
|
||
srt_path = combo_dir / "transcript.srt"
|
||
cues = parse_srt(srt_path.read_text(encoding="utf-8"))
|
||
return {"tag": tag, "model": model_path, "elapsed": round(elapsed, 1), "cues": cues}
|
||
|
||
|
||
def main() -> None:
|
||
parser = argparse.ArgumentParser(description=__doc__)
|
||
parser.add_argument("--video", default=VIDEO_PATH, help="待测视频路径")
|
||
parser.add_argument("--out", default=str(OUT_DIR), help="实验输出目录")
|
||
args = parser.parse_args()
|
||
|
||
video = Path(args.video)
|
||
out_dir = Path(args.out)
|
||
out_dir.mkdir(parents=True, exist_ok=True)
|
||
if not video.is_file():
|
||
sys.exit(f"视频不存在: {video}")
|
||
# 用 ffmpeg -i 的 stderr 探测视频时长（ffprobe 可能不在 PATH）。
|
||
probe = subprocess.run(
|
||
[_ffmpeg_bin(), "-i", str(video)],
|
||
capture_output=True, text=True,
|
||
).stderr
|
||
m = re.search(r"Duration:\s*(\d+):(\d+):(\d+)\.(\d+)", probe)
|
||
if m:
|
||
total_seconds = int(m[1]) * 3600 + int(m[2]) * 60 + int(m[3]) + int(m[4]) / 100
|
||
else:
|
||
total_seconds = 0.0
|
||
|
||
print(f"视频: {video} 时长 {total_seconds:.1f}s")
|
||
print("提取 16k 单声道音频 ...")
|
||
wav = extract_audio(video, out_dir)
|
||
|
||
# 4 个组合：v3/v2 × 生产VAD/无VAD，串行执行（模型加载复用由 faster-whisper
|
||
# 内部承担，这里每次 invoke 新载一次，串行避免显存竞争）。
|
||
combos = [
|
||
("v3_vad", "faster-whisper-large-v3", True, PROD_VAD_PARAMS),
|
||
("v3_novad", "faster-whisper-large-v3", False, None),
|
||
("v2_vad", "faster-whisper-large-v2", True, PROD_VAD_PARAMS),
|
||
("v2_novad", "faster-whisper-large-v2", False, None),
|
||
]
|
||
results = []
|
||
for tag, model, vad_filter, vp in combos:
|
||
print(f"\n=== {tag} (model={model}, vad_filter={vad_filter}) ===")
|
||
r = run_combo(tag, model, wav, out_dir, vad_filter, vp)
|
||
s = stats(r["cues"], total_seconds)
|
||
r.update(s)
|
||
results.append(r)
|
||
# 简要打印每条产物前若干行验证非空。
|
||
print(f" {s['count']} 条 / 展示和 {s['display_sum']}s / "
|
||
f"覆盖 {s['union']}s ({s['coverage_pct']}%) / "
|
||
f"幻觉长段 {s['halluc_count']} 条 {s['halluc_seconds']}s "
|
||
f"/ 耗时 {r['elapsed']}s")
|
||
|
||
# 汇总表：与控制台对齐输出 CSV + 终端 markdown 表格。
|
||
csv_path = out_dir / "summary.csv"
|
||
with csv_path.open("w", newline="", encoding="utf-8") as f:
|
||
writer = csv.writer(f)
|
||
writer.writerow(["tag", "model", "count", "display_sum_s", "union_s",
|
||
"coverage_pct", "halluc_count", "halluc_seconds_s",
|
||
"elapsed_s"])
|
||
for r in results:
|
||
writer.writerow([r["tag"], r["model"], r["count"], r["display_sum"],
|
||
r["union"], r["coverage_pct"], r["halluc_count"],
|
||
r["halluc_seconds"], r["elapsed"]])
|
||
|
||
print("\n===== 汇总（大模型实物对比，同代码同配置） =====")
|
||
print(f"{'tag':9s} {'条数':>5s} {'展示和s':>8s} {'覆盖s':>7s} "
|
||
f"{'覆盖%':>6s} {'幻觉段数':>7s} {'幻觉s':>7s} {'耗时s':>7s}")
|
||
for r in results:
|
||
print(f"{r['tag']:9s} {r['count']:5d} {r['display_sum']:8.1f} "
|
||
f"{r['union']:7.1f} {r['coverage_pct']:6.1f} "
|
||
f"{r['halluc_count']:7d} {r['halluc_seconds']:7.1f} "
|
||
f"{r['elapsed']:7.1f}")
|
||
print(f"\n历史 large-v3 文档值（调研文档 2026-09）："
|
||
f"生产VAD 115 条/621s 覆盖；无VAD 369 条/983s 覆盖（有 119-149s、"
|
||
f"600-630s 30s 幻觉长段）。")
|
||
print(f"汇总已存: {csv_path}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main() |