"""whisper large-v3 vs large-v2 同配置对比实验脚本(savr-1054 全片)。 目的:验证 large-v2 是否比当前生产用的 large-v3 更适合本项目(日语成人内容, 呻吟/BGM 混叠、弱语音)。方法:对同一视频,用相同参数分别跑 large-v3 与 large-v2 的「生产 VAD」与「无 VAD」两条链路,统计条数/覆盖时长/幻觉长段等 指标,与 docs/调研-whisper漏句与decode_full验证.md 中记录的 v3 历史结果对照。 复现路径:直接调用 nodes/whisper.py 的 invoke(真实节点代码),不绕开注册表 或重写业务逻辑。显式传 vad_parameters 复现文档实验一的"生产 VAD (threshold0.5/ms1000/pad200)",并关闭自动 VAD(WOV_AUTO_VAD=0),确保 v3/v2 差异纯粹来自模型,而非信号分析分支抖动。 用法: uv run python scripts/compare_whisper_v2_vs_v3.py \ --video /mnt/fnOS/123/savr-1054/4k2.me@savr01054_2_8k.mp4 \ --out data/experiments/whisper_v2_vs_v3 产物: /audio.wav 16k 单声道(ffmpeg 提取一次,复用) //transcript.srt 各组合的 SRT 产物 /summary.csv 指标汇总 控制台打印对比表 """ from __future__ import annotations import argparse import csv import os import re import subprocess import sys import time from pathlib import Path # 保证脚本可从仓库根目录直接 import nodes/wov_sdk。 ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT)) os.environ.setdefault("WOV_AUTO_VAD", "0") # 关闭自动 VAD,用显式参数 from nodes.ffmpeg import _ffmpeg_bin # noqa: E402 from nodes.subtitle_cleanup import JAPANESE_HALLUCINATION_TOKENS # noqa: E402 from wov_sdk.models import InvokeRequest # noqa: E402 # 实验一的生产 VAD 配置(文档记录):threshold0.5 / min_silence1000 / pad200。 PROD_VAD_PARAMS = { "threshold": 0.5, "min_silence_duration_ms": 1000, "speech_pad_ms": 200, } # 幻觉长段判定:展示时长 ≥ 阈值 且文本命中日文寒暄词表(与 subtitle_cleanup # 的 decode_full 清洗同一判据,这里不删除只统计)。 HALLUC_THRESHOLD_SECONDS = 15.0 VIDEO_PATH = "/mnt/fnOS/123/savr-1054/4k2.me@savr01054_2_8k.mp4" OUT_DIR = ROOT / "data" / "experiments" / "whisper_v2_vs_v3" def extract_audio(video: Path, out_dir: Path) -> Path: """用 ffmpeg 提取 16k 单声道 wav(与 ffmpeg-extract 节点相同规格)。""" wav = out_dir / "audio.wav" if wav.is_file() and wav.stat().st_size > 0: return wav subprocess.run( [ _ffmpeg_bin(), "-y", "-i", str(video), "-vn", "-ac", "1", "-ar", "16000", str(wav), ], check=True, capture_output=True, ) return wav def parse_srt(srt_text: str) -> list[dict]: """解析 SRT 文本为 cue 列表(序号/起止/文本/时长),文档统计口径。""" cues = [] # 标准 SRT 块:序号 / 时间轴 / 文本行(可多行)/ 空行。 blocks = re.split(r"\n\s*\n", srt_text.strip()) for block in blocks: lines = [ln for ln in block.splitlines() if ln.strip()] if len(lines) < 2: continue if not lines[0].strip().isdigit(): continue m = re.match( r"(\d+):(\d+):(\d+)[,.](\d+)\s*-->\s*(\d+):(\d+):(\d+)[,.](\d+)", lines[1], ) if not m: continue start = (int(m[1]) * 3600 + int(m[2]) * 60 + int(m[3])) + int(m[4]) / 1000 end = (int(m[5]) * 3600 + int(m[6]) * 60 + int(m[7])) + int(m[8]) / 1000 text = " ".join(lines[2:]).strip() cues.append({ "start": start, "end": end, "duration": end - start, "text": text, }) return cues def stats(cues: list[dict], total_seconds: float) -> dict: """汇总指标:条数/展示时长和/覆盖时长(并集)/幻觉长段。""" count = len(cues) display_sum = sum(c["duration"] for c in cues) # 覆盖并集:按开始时间排序后合并重叠区间。 ordered = sorted(cues, key=lambda c: c["start"]) union = 0.0 cur_s, cur_e = None, None for c in ordered: if cur_s is None: cur_s, cur_e = c["start"], c["end"] elif c["start"] <= cur_e: cur_e = max(cur_e, c["end"]) else: union += cur_e - cur_s cur_s, cur_e = c["start"], c["end"] if cur_s is not None: union += cur_e - cur_s # 幻觉长段:时长≥阈值 且 文本含任一寒暄词。 halluc = [ c for c in cues if c["duration"] >= HALLUC_THRESHOLD_SECONDS and any(tok in c["text"] for tok in JAPANESE_HALLUCINATION_TOKENS) ] halluc_seconds = sum(c["duration"] for c in halluc) return { "count": count, "display_sum": round(display_sum, 1), "union": round(union, 1), "coverage_pct": round(union / total_seconds * 100, 1) if total_seconds else 0.0, "halluc_count": len(halluc), "halluc_seconds": round(halluc_seconds, 1), } def run_combo(tag: str, model_path: str, wav: Path, out_dir: Path, vad_filter: bool, vad_parameters: dict | None) -> dict: """跑一趟 whisper.invoke(真实节点路径),返回统计与产物路径。""" combo_dir = out_dir / tag combo_dir.mkdir(parents=True, exist_ok=True) params: dict = { "language": "ja", "model_path": model_path, "condition_on_previous_text": False, "chunk_seconds": 60, "vad_filter": vad_filter, "beam_size": 1, } if vad_parameters is not None: params["vad_parameters"] = vad_parameters request = InvokeRequest( run_id=f"exp-{tag}", node_instance_id="whisper", inputs={"audio_uri": str(wav)}, params=params, output_dir=str(combo_dir), ) started = time.monotonic() from nodes.whisper import invoke response = invoke(request) elapsed = time.monotonic() - started if response.status != "completed": raise RuntimeError(f"[{tag}] invoke failed: {response.error}") srt_path = combo_dir / "transcript.srt" cues = parse_srt(srt_path.read_text(encoding="utf-8")) return {"tag": tag, "model": model_path, "elapsed": round(elapsed, 1), "cues": cues} def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--video", default=VIDEO_PATH, help="待测视频路径") parser.add_argument("--out", default=str(OUT_DIR), help="实验输出目录") args = parser.parse_args() video = Path(args.video) out_dir = Path(args.out) out_dir.mkdir(parents=True, exist_ok=True) if not video.is_file(): sys.exit(f"视频不存在: {video}") # 用 ffmpeg -i 的 stderr 探测视频时长(ffprobe 可能不在 PATH)。 probe = subprocess.run( [_ffmpeg_bin(), "-i", str(video)], capture_output=True, text=True, ).stderr m = re.search(r"Duration:\s*(\d+):(\d+):(\d+)\.(\d+)", probe) if m: total_seconds = int(m[1]) * 3600 + int(m[2]) * 60 + int(m[3]) + int(m[4]) / 100 else: total_seconds = 0.0 print(f"视频: {video} 时长 {total_seconds:.1f}s") print("提取 16k 单声道音频 ...") wav = extract_audio(video, out_dir) # 4 个组合:v3/v2 × 生产VAD/无VAD,串行执行(模型加载复用由 faster-whisper # 内部承担,这里每次 invoke 新载一次,串行避免显存竞争)。 combos = [ ("v3_vad", "faster-whisper-large-v3", True, PROD_VAD_PARAMS), ("v3_novad", "faster-whisper-large-v3", False, None), ("v2_vad", "faster-whisper-large-v2", True, PROD_VAD_PARAMS), ("v2_novad", "faster-whisper-large-v2", False, None), ] results = [] for tag, model, vad_filter, vp in combos: print(f"\n=== {tag} (model={model}, vad_filter={vad_filter}) ===") r = run_combo(tag, model, wav, out_dir, vad_filter, vp) s = stats(r["cues"], total_seconds) r.update(s) results.append(r) # 简要打印每条产物前若干行验证非空。 print(f" {s['count']} 条 / 展示和 {s['display_sum']}s / " f"覆盖 {s['union']}s ({s['coverage_pct']}%) / " f"幻觉长段 {s['halluc_count']} 条 {s['halluc_seconds']}s " f"/ 耗时 {r['elapsed']}s") # 汇总表:与控制台对齐输出 CSV + 终端 markdown 表格。 csv_path = out_dir / "summary.csv" with csv_path.open("w", newline="", encoding="utf-8") as f: writer = csv.writer(f) writer.writerow(["tag", "model", "count", "display_sum_s", "union_s", "coverage_pct", "halluc_count", "halluc_seconds_s", "elapsed_s"]) for r in results: writer.writerow([r["tag"], r["model"], r["count"], r["display_sum"], r["union"], r["coverage_pct"], r["halluc_count"], r["halluc_seconds"], r["elapsed"]]) print("\n===== 汇总(大模型实物对比,同代码同配置) =====") print(f"{'tag':9s} {'条数':>5s} {'展示和s':>8s} {'覆盖s':>7s} " f"{'覆盖%':>6s} {'幻觉段数':>7s} {'幻觉s':>7s} {'耗时s':>7s}") for r in results: print(f"{r['tag']:9s} {r['count']:5d} {r['display_sum']:8.1f} " f"{r['union']:7.1f} {r['coverage_pct']:6.1f} " f"{r['halluc_count']:7d} {r['halluc_seconds']:7.1f} " f"{r['elapsed']:7.1f}") print(f"\n历史 large-v3 文档值(调研文档 2026-09):" f"生产VAD 115 条/621s 覆盖;无VAD 369 条/983s 覆盖(有 119-149s、" f"600-630s 30s 幻觉长段)。") print(f"汇总已存: {csv_path}") if __name__ == "__main__": main()