cat-shark
f656ec98c5
模型与管线切换后需要把媒体库里旧管线生成的字幕整批重跑。批量引擎按"视频旁 已有字幕即 SKIPPED"判定，重跑前必须先统计范围、再把旧字幕改名备份，否则建出来 的任务会把所有视频全部跳过。 - `scripts/plan_regenerate_subtitles.py`：扫描媒体库，按 CN 产物的最早 mtime 判定 生成时间，统计待重生成的数量/时长/体积并输出 JSON 计划；`--backup-old [--apply] [--select]` 把旧字幕原地改名为 `<原名>.old-<日期>` 供批量重跑。脚本不依赖仓库， 可拷到媒体库主机直接跑（CIFS 挂载下逐文件 ffprobe 太慢）。 - 测试覆盖真实 10 秒视频与真实字幕文件，含 ffprobe 时长探测、分类边界、 备份幂等与选择清单解析。
321 lines
13 KiB
Python
321 lines
13 KiB
Python
"""统计媒体库中需要重新生成字幕的视频数量与时长，产出可复用的重生成计划。
|
||
|
||
判定口径（`--before` 可调）：
|
||
|
||
- 只看本流水线产出的 CN 字幕：`<视频名>.CN.srt` 与 `<视频名>.CN_dual_eye.ass`；
|
||
- 生成时间取这些产物中**最早**的 mtime。双目 `.ass` 可能被样式统一脚本原地
|
||
改写而"变新"，中文字幕的 mtime 才反映真实生成时间，取最早值两者兼容；
|
||
- 生成时间早于 `--before`（默认 2026-09-01，本地时区）即列为待重新生成；
|
||
- 完全没有 CN 字幕的视频单独统计，不算"重生成"（那属于首次生成）。
|
||
|
||
脚本不依赖仓库，可拷到媒体库所在主机直接跑（共享目录挂载方式下逐文件
|
||
ffprobe 太慢）：目录内容只读，不改动任何文件；结果写 JSON 计划供批量重跑。
|
||
|
||
用法：
|
||
python3 scripts/plan_regenerate_subtitles.py /mnt/fnOS/123
|
||
python3 scripts/plan_regenerate_subtitles.py /vol1/1000/123 --before 2026-09-01 \
|
||
--out data/experiments/regen_plan.json
|
||
|
||
批量引擎看到视频旁已有字幕就会跳过该视频，重生成前需先改名备份旧字幕：
|
||
|
||
python3 scripts/plan_regenerate_subtitles.py /vol1/1000/123 --backup-old
|
||
python3 scripts/plan_regenerate_subtitles.py /vol1/1000/123 --backup-old --apply \
|
||
--select sample.txt
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import subprocess
|
||
import sys
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
|
||
# 与 src/wov_app/batch.py 的 VIDEO_EXTENSIONS / SUBTITLE_EXTENSIONS 保持一致，
|
||
# 本脚本要在媒体库主机上独立运行，故不复用仓库常量。
|
||
VIDEO_EXTENSIONS = {
|
||
".mp4", ".mkv", ".avi", ".mov", ".webm", ".flv", ".ts",
|
||
".m4v", ".wmv", ".mpg", ".mpeg", ".3gp",
|
||
}
|
||
SUBTITLE_EXTENSIONS = {".srt", ".ass", ".ssa", ".vtt"}
|
||
|
||
# CN 产物识别：这两个后缀由本流水线生成（见 batch.py 的产物命名约定）；
|
||
# 媒体库里其它 .srt 多为片源自带字幕，不参与"字幕生成时间"判定。
|
||
CN_PRODUCT_MARKERS = (".cn.srt", ".cn_dual_eye.ass")
|
||
|
||
PENDING = "pending"
|
||
CURRENT = "current"
|
||
MISSING = "missing"
|
||
|
||
|
||
def iter_videos(root: Path) -> list[Path]:
|
||
"""递归列出媒体库下全部视频文件，按路径排序保证结果可复现。"""
|
||
paths = [
|
||
p for p in root.rglob("*")
|
||
if p.is_file() and p.suffix.lower() in VIDEO_EXTENSIONS
|
||
]
|
||
return sorted(paths)
|
||
|
||
|
||
def cn_products(video: Path) -> list[Path]:
|
||
"""列出视频旁由本流水线产出的 CN 字幕（同一目录、文件名含视频主名）。"""
|
||
stem = video.stem.lower()
|
||
found: list[Path] = []
|
||
for item in video.parent.iterdir():
|
||
if not item.is_file():
|
||
continue
|
||
name = item.name.lower()
|
||
if item.suffix.lower() not in SUBTITLE_EXTENSIONS:
|
||
continue
|
||
if stem in name and name.endswith(CN_PRODUCT_MARKERS):
|
||
found.append(item)
|
||
return sorted(found)
|
||
|
||
|
||
def sidecar_subtitles(video: Path) -> list[Path]:
|
||
"""列出批量引擎认定为“已有字幕”的旁挂字幕文件（与 batch.py 判定一致）。
|
||
|
||
同目录、字幕扩展名、文件名包含视频主名；主名只有单个字符时只接受
|
||
“主名.”前缀，避免 a.mp4 误配 apple.srt。
|
||
"""
|
||
stem = video.stem.lower()
|
||
try:
|
||
siblings = list(video.parent.iterdir())
|
||
except OSError:
|
||
return []
|
||
found: list[Path] = []
|
||
for item in siblings:
|
||
try:
|
||
if not item.is_file() or item.suffix.lower() not in SUBTITLE_EXTENSIONS:
|
||
continue
|
||
except OSError:
|
||
continue
|
||
name = item.name.lower()
|
||
if len(stem) <= 1:
|
||
matched = name.startswith(stem + ".")
|
||
else:
|
||
matched = stem in name
|
||
if matched:
|
||
found.append(item)
|
||
return sorted(found)
|
||
|
||
|
||
def read_select_list(path: Path) -> set[str]:
|
||
"""读取 `--select` 文件里的视频标识（每行一个，支持行首或行尾 `#` 注释）。"""
|
||
keys: set[str] = set()
|
||
for line in Path(path).read_text(encoding="utf-8").splitlines():
|
||
text = line.split(" #", 1)[0].strip()
|
||
if text and not text.startswith("#"):
|
||
keys.add(text)
|
||
return keys
|
||
|
||
|
||
def backup_old_subtitles(items: list[dict], suffix: str, apply: bool = False,
|
||
select: set[str] | None = None) -> list[tuple[Path, Path]]:
|
||
"""把待重生成视频的旁挂字幕原地改名，让批量引擎不再把该视频判为已有字幕。
|
||
|
||
改名目标为 `<原名><suffix>`；目标已存在（同一批重复执行）时跳过，不覆盖
|
||
旧备份。apply=False 只返回改名为计划，不动磁盘。select 按绝对路径、
|
||
相对路径或文件名筛选要处理的视频。
|
||
"""
|
||
changed: list[tuple[Path, Path]] = []
|
||
for item in items:
|
||
video = Path(item["video"])
|
||
if select is not None:
|
||
keys = (str(video), item.get("rel"), video.name)
|
||
if not any(key in select for key in keys if key):
|
||
continue
|
||
for sub in sidecar_subtitles(video):
|
||
target = sub.with_name(sub.name + suffix)
|
||
if target.exists():
|
||
continue
|
||
if apply:
|
||
sub.rename(target)
|
||
changed.append((sub, target))
|
||
return changed
|
||
|
||
|
||
def probe_duration(video: Path, timeout: float = 120.0) -> float | None:
|
||
"""用 ffprobe 读取容器时长（秒）；失败返回 None，由调用方单独计数。"""
|
||
try:
|
||
proc = subprocess.run(
|
||
[
|
||
"ffprobe", "-v", "error",
|
||
"-show_entries", "format=duration",
|
||
"-of", "default=nw=1:nk=1", str(video),
|
||
],
|
||
capture_output=True, text=True, timeout=timeout,
|
||
)
|
||
except (OSError, subprocess.TimeoutExpired):
|
||
return None
|
||
if proc.returncode != 0:
|
||
return None
|
||
try:
|
||
return float(proc.stdout.strip())
|
||
except ValueError:
|
||
return None
|
||
|
||
|
||
def build_items(root: Path, workers: int, probe: bool = True) -> list[dict]:
|
||
"""扫描全部视频，逐条产出路径、大小、时长与 CN 字幕生成时间。"""
|
||
videos = iter_videos(root)
|
||
durations: list[float | None] = [None] * len(videos)
|
||
if probe and videos:
|
||
with ThreadPoolExecutor(max_workers=workers) as pool:
|
||
durations = list(pool.map(probe_duration, videos))
|
||
|
||
items: list[dict] = []
|
||
for video, duration in zip(videos, durations):
|
||
products = cn_products(video)
|
||
mtimes = [p.stat().st_mtime for p in products]
|
||
items.append({
|
||
"video": str(video),
|
||
"rel": str(video.relative_to(root)),
|
||
"size": video.stat().st_size,
|
||
"duration": duration,
|
||
"generated_at": datetime.fromtimestamp(min(mtimes)).isoformat() if mtimes else None,
|
||
"cn_products": [
|
||
{"path": str(p), "mtime": datetime.fromtimestamp(p.stat().st_mtime).isoformat()}
|
||
for p in products
|
||
],
|
||
})
|
||
return items
|
||
|
||
|
||
def classify(items: list[dict], before: datetime) -> dict[str, list[dict]]:
|
||
"""按 CN 字幕生成时间把视频分为待重生成/已最新/无 CN 字幕三组。"""
|
||
groups: dict[str, list[dict]] = {PENDING: [], CURRENT: [], MISSING: []}
|
||
for item in items:
|
||
if item["generated_at"] is None:
|
||
groups[MISSING].append(item)
|
||
elif datetime.fromisoformat(item["generated_at"]) < before:
|
||
groups[PENDING].append(item)
|
||
else:
|
||
groups[CURRENT].append(item)
|
||
return groups
|
||
|
||
|
||
def _agg(items: list[dict]) -> dict:
|
||
"""汇总一组的数量、总时长、总大小，时长缺失的视频单独计数。"""
|
||
return {
|
||
"count": len(items),
|
||
"seconds": sum(i["duration"] for i in items if i["duration"] is not None),
|
||
"bytes": sum(i["size"] for i in items),
|
||
"no_duration": sum(1 for i in items if i["duration"] is None),
|
||
}
|
||
|
||
|
||
def _fmt_hours(seconds: float) -> str:
|
||
return f"{seconds / 3600:.2f} 小时"
|
||
|
||
|
||
def _fmt_duration(probed: bool, seconds: float) -> str:
|
||
"""未探测时长时（--backup-old 等只统计数量的场景）不显示 0 小时。"""
|
||
return _fmt_hours(seconds) if probed else "未探测时长"
|
||
|
||
|
||
def _fmt_size(num_bytes: int) -> str:
|
||
return f"{num_bytes / (1024 ** 3):.1f} GiB"
|
||
|
||
|
||
def format_summary(root: Path, items: list[dict], groups: dict[str, list[dict]],
|
||
before: datetime) -> str:
|
||
"""生成人类可读的统计报告（数量 / 时长 / 体积 / 按月分布）。"""
|
||
probed = any(i["duration"] is not None for i in items)
|
||
lines = [
|
||
f"媒体库: {root}",
|
||
f"视频总数: {len(items)} 个 / {_fmt_duration(probed, _agg(items)['seconds'])} / "
|
||
f"{_fmt_size(_agg(items)['bytes'])}",
|
||
"",
|
||
f"待重新生成（CN 字幕早于 {before:%Y-%m-%d}）: ",
|
||
]
|
||
pending = _agg(groups[PENDING])
|
||
duration_text = _fmt_duration(probed, pending["seconds"])
|
||
lines.append(f" {pending['count']} 个 / {duration_text} / {_fmt_size(pending['bytes'])}")
|
||
|
||
by_month: dict[str, dict] = {}
|
||
for item in groups[PENDING]:
|
||
month = item["generated_at"][:7]
|
||
bucket = by_month.setdefault(month, {"count": 0, "seconds": 0.0})
|
||
bucket["count"] += 1
|
||
bucket["seconds"] += item["duration"] or 0.0
|
||
for month in sorted(by_month):
|
||
bucket = by_month[month]
|
||
lines.append(f" {month}: {bucket['count']} 个 / "
|
||
f"{_fmt_duration(probed, bucket['seconds'])}")
|
||
|
||
current = _agg(groups[CURRENT])
|
||
lines.append("")
|
||
lines.append(f"已是最新（生成时间 >= {before:%Y-%m-%d}）: {current['count']} 个 / "
|
||
f"{_fmt_duration(probed, current['seconds'])} / {_fmt_size(current['bytes'])}")
|
||
missing = _agg(groups[MISSING])
|
||
lines.append(f"无 CN 字幕（首次生成）: {missing['count']} 个 / "
|
||
f"{_fmt_duration(probed, missing['seconds'])} / {_fmt_size(missing['bytes'])}")
|
||
|
||
no_duration = sum(1 for i in items if i["duration"] is None)
|
||
if no_duration and probed:
|
||
lines.append(f"警告: {no_duration} 个视频未能读出时长，未计入时长合计")
|
||
return "\n".join(lines)
|
||
|
||
|
||
def main(argv: list[str] | None = None) -> int:
|
||
parser = argparse.ArgumentParser(description="统计需要重新生成字幕的视频数量与时长")
|
||
parser.add_argument("root", help="媒体库根目录")
|
||
parser.add_argument("--before", default="2026-09-01",
|
||
help="重生成分界线，早于该日期生成的 CN 字幕待重跑（默认 2026-09-01）")
|
||
parser.add_argument("--out", help="计划 JSON 输出路径（默认只打印统计）")
|
||
parser.add_argument("--workers", type=int, default=8, help="ffprobe 并发数（默认 8）")
|
||
parser.add_argument("--no-probe", action="store_true", help="跳过时长探测（只统计数量）")
|
||
parser.add_argument("--list-pending", action="store_true", help="额外打印待重生成的视频路径")
|
||
parser.add_argument("--backup-old", action="store_true",
|
||
help="把待重生成视频的旁挂字幕改名备份（默认预览，需 --apply 落盘）")
|
||
parser.add_argument("--apply", action="store_true", help="与 --backup-old 一起用时真正改名")
|
||
parser.add_argument("--select", help="只处理该文件列出的视频（每行一个路径或文件名）")
|
||
parser.add_argument("--suffix", help="备份后缀，默认 .old-<当天日期>")
|
||
args = parser.parse_args(argv)
|
||
|
||
root = Path(args.root).expanduser().resolve()
|
||
if not root.is_dir():
|
||
print(f"目录不存在: {root}", file=sys.stderr)
|
||
return 2
|
||
before = datetime.fromisoformat(args.before)
|
||
|
||
need_duration = args.out is not None or args.list_pending or not args.backup_old
|
||
items = build_items(root, max(1, args.workers), probe=need_duration and not args.no_probe)
|
||
groups = classify(items, before)
|
||
print(format_summary(root, items, groups, before))
|
||
if args.list_pending:
|
||
print("\n待重新生成的视频:")
|
||
for item in groups[PENDING]:
|
||
print(f" {item['video']}")
|
||
|
||
if args.backup_old:
|
||
suffix = args.suffix or f".old-{datetime.now():%Y%m%d}"
|
||
select = read_select_list(Path(args.select)) if args.select else None
|
||
changed = backup_old_subtitles(groups[PENDING], suffix, apply=args.apply, select=select)
|
||
verb = "已改名" if args.apply else "待改名（预览，未落盘）"
|
||
print(f"\n旧字幕备份（后缀 {suffix}）: {verb} {len(changed)} 个文件")
|
||
for source, target in changed:
|
||
print(f" {source} -> {target.name}")
|
||
|
||
if args.out:
|
||
out = Path(args.out).expanduser()
|
||
out.parent.mkdir(parents=True, exist_ok=True)
|
||
status_of = {id(i): name for name, group in groups.items() for i in group}
|
||
payload = {
|
||
"root": str(root),
|
||
"before": before.isoformat(),
|
||
"scanned_at": datetime.now().isoformat(),
|
||
"totals": {name: _agg(group) for name, group in groups.items()},
|
||
"items": [{**i, "status": status_of[id(i)]} for i in items],
|
||
}
|
||
out.write_text(json.dumps(payload, ensure_ascii=False, indent=1), encoding="utf-8")
|
||
print(f"\n计划 JSON 已写入: {out}")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|