feat: 任务断点续跑/暂停中断/LLM 过滤优化与调度容错
调度与状态机: - 修复 PAUSED 任务被拾起后复活执行(点击暂停反而开始任务):next_queued_run 只取 QUEUED,execute_run 以 PAUSED 进入直接返回,暂停必须显式 resume - 重启恢复:启动时 recover_interrupted_runs 把遗留 RUNNING 置 QUEUED(保留产物) - 暂停信号 paused.flag:暂停接口写、继续/重试清除,OCR 逐帧检查秒级中断, 节点内被暂停保持 PAUSED 不误报 FAILED - 调度轮询容错:_loop 异常不杀死线程(曾致任务永久停留 QUEUED) subtitle-ocr 节点级断点: - ocr_partial.jsonl 逐帧存档,重启/暂停后只处理未处理帧,产物与一次跑完一致 - 进度日志携带窗口平均耗时与线程数;取消后抑制进度日志井喷 llm-filter 过滤质量与限流自适应: - 上下文净化:喂给 LLM 的是过滤后的字幕(规则层垃圾从上下文剔除) - 正则确定性过滤:裸网址域名、HTML/水印模式直接删除 - 429/5xx 指数退避重试;worker 限流错误 report_failure 内存临时降最大线程数 并缩容(无错误窗口回升),失败条目降并发后重试一轮 - 保留长文本保护(noise 不删 ≥min_keep_len 文本,LLM 判定不稳的必要兜底) 前端: - 工作流编排页支持选择工作流编辑(加载最新/历史版本)、版本历史面板、 新建/编辑双模式;管理后台编辑跳转 workflow.html?edit=<id> 工作流:ocr-subtitle v7(filter pool_max_workers=20、pool_fast_threshold=1)
This commit is contained in:
+22
-2
@@ -303,18 +303,38 @@ class Database:
|
||||
conn.execute("DELETE FROM artifacts WHERE run_id = ?", (run_id,))
|
||||
|
||||
def next_queued_run(self) -> dict[str, Any] | None:
|
||||
"""按创建时间返回最早一条可执行任务(排队或已暂停待续跑)。"""
|
||||
"""按创建时间返回最早一条排队(QUEUED)任务。
|
||||
|
||||
只取 QUEUED:PAUSED 任务必须由用户显式 resume(转回 QUEUED)后调度器
|
||||
才重新执行。修复回归——此前把 PAUSED 也当可执行任务拾起,execute_run
|
||||
会先置 RUNNING 再检查暂停,导致"点击暂停反而开始任务"。
|
||||
"""
|
||||
with self._connect() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT * FROM workflow_runs
|
||||
WHERE status IN ('QUEUED', 'PAUSED')
|
||||
WHERE status = 'QUEUED'
|
||||
ORDER BY created_at ASC
|
||||
LIMIT 1
|
||||
"""
|
||||
).fetchone()
|
||||
return self._parse_overrides(row) if row else None
|
||||
|
||||
def recover_interrupted_runs(self, updated_at: str) -> int:
|
||||
"""重启恢复:把遗留 RUNNING 任务恢复为 QUEUED,返回恢复数量。
|
||||
|
||||
进程被杀/重启时 RUNNING 任务没有机会收尾:保持 RUNNING 会永久孤儿
|
||||
(next_queued_run 不拾起)。恢复为 QUEUED 后调度器会从产物表
|
||||
(restore_run_outputs)断点续跑,不重复已完成节点;用户主动暂停的
|
||||
PAUSED 任务保持不变,等待显式 resume。
|
||||
"""
|
||||
with self._connect() as conn:
|
||||
cur = conn.execute(
|
||||
"UPDATE workflow_runs SET status = 'QUEUED', updated_at = ? WHERE status = 'RUNNING'",
|
||||
(updated_at,),
|
||||
)
|
||||
return cur.rowcount
|
||||
|
||||
def pause_run(self, run_id: str, updated_at: str) -> None:
|
||||
"""暂停任务:置为 PAUSED;调度器会在节点边界检查并停止推进。"""
|
||||
with self._connect() as conn:
|
||||
|
||||
+4
-1
@@ -13,6 +13,7 @@ load_dotenv()
|
||||
|
||||
import os # noqa: E402
|
||||
from contextlib import asynccontextmanager # noqa: E402
|
||||
from datetime import datetime, timezone # noqa: E402
|
||||
from pathlib import Path # noqa: E402
|
||||
|
||||
from fastapi import FastAPI # noqa: E402
|
||||
@@ -38,7 +39,9 @@ async def lifespan(app: FastAPI):
|
||||
# 默认创建演示工作流,可关闭便于测试。
|
||||
if os.getenv("WOV_AUTO_SEED", "1") == "1":
|
||||
seed_default_workflows(db)
|
||||
|
||||
# 重启恢复:上次进程被杀时遗留的 RUNNING 任务恢复为 QUEUED,
|
||||
# 调度器会从产物表断点续跑(不重复已完成节点);PAUSED 保持等待显式 resume。
|
||||
db.recover_interrupted_runs(datetime.now(timezone.utc).isoformat())
|
||||
scheduler = WorkflowScheduler(db, STORAGE_DIR)
|
||||
# 调度器默认开启,处理排队中的任务;测试可关闭后手动执行。
|
||||
if os.getenv("WOV_SCHEDULER_ENABLED", "1") == "1":
|
||||
|
||||
@@ -141,6 +141,10 @@ def retry_run(run_id: str, db: Database = Depends(_get_db)) -> dict:
|
||||
raise HTTPException(status_code=422, detail="only failed runs can be retried")
|
||||
# reset_run 会清空进度、错误和旧产物,确保从头开始。
|
||||
db.reset_run(run_id, _now_iso())
|
||||
# 重试前清除可能残留的暂停信号(任务失败时信号文件可能仍在)。
|
||||
from wov_app.config import STORAGE_DIR
|
||||
|
||||
(STORAGE_DIR / "runs" / run_id / "paused.flag").unlink(missing_ok=True)
|
||||
return {"id": run_id, "status": "QUEUED"}
|
||||
|
||||
|
||||
@@ -153,6 +157,13 @@ def pause_run(run_id: str, db: Database = Depends(_get_db)) -> dict:
|
||||
if run["status"] not in ("QUEUED", "RUNNING"):
|
||||
raise HTTPException(status_code=422, detail="only queued or running runs can be paused")
|
||||
db.pause_run(run_id, _now_iso())
|
||||
# 写入暂停信号文件:运行中的节点(如 OCR)逐帧检查到后立即中止,
|
||||
# 由调度器保持 PAUSED;resume 时清除。
|
||||
from wov_app.config import STORAGE_DIR
|
||||
|
||||
run_dir = STORAGE_DIR / "runs" / run_id
|
||||
run_dir.mkdir(parents=True, exist_ok=True)
|
||||
(run_dir / "paused.flag").write_text("", encoding="utf-8")
|
||||
return {"id": run_id, "status": "PAUSED"}
|
||||
|
||||
|
||||
@@ -165,6 +176,10 @@ def resume_run(run_id: str, db: Database = Depends(_get_db)) -> dict:
|
||||
if run["status"] != "PAUSED":
|
||||
raise HTTPException(status_code=422, detail="only paused runs can be resumed")
|
||||
db.resume_run(run_id, _now_iso())
|
||||
# 清除暂停信号文件,避免节点误判仍处于暂停状态。
|
||||
from wov_app.config import STORAGE_DIR
|
||||
|
||||
(STORAGE_DIR / "runs" / run_id / "paused.flag").unlink(missing_ok=True)
|
||||
return {"id": run_id, "status": "QUEUED"}
|
||||
|
||||
@router.delete("/api/runs/{run_id}")
|
||||
|
||||
@@ -96,12 +96,18 @@ class WorkflowScheduler:
|
||||
def _loop(self) -> None:
|
||||
"""轮询循环:有排队任务就立即执行,否则休眠一个间隔。"""
|
||||
while not self._stopping:
|
||||
run = self.db.next_queued_run()
|
||||
if run is not None:
|
||||
self.execute_run(run["id"])
|
||||
else:
|
||||
try:
|
||||
run = self.db.next_queued_run()
|
||||
if run is not None:
|
||||
self.execute_run(run["id"])
|
||||
else:
|
||||
time.sleep(self.interval_seconds)
|
||||
except Exception: # noqa: BLE001
|
||||
# 单次轮询异常不杀死调度线程:曾因 next_queued_run/execute_run
|
||||
# 的未捕获异常导致线程退出,任务永远停留在 QUEUED 不被拾起
|
||||
# (run_011d01f19999 实际发生)。记录后跳过本轮,下一轮继续。
|
||||
logger.exception("调度器轮询异常,跳过本轮")
|
||||
time.sleep(self.interval_seconds)
|
||||
|
||||
def _resolve_ref(
|
||||
self,
|
||||
ref: str,
|
||||
@@ -124,6 +130,12 @@ class WorkflowScheduler:
|
||||
# 任务不存在或不在可执行状态(排队/暂停)时直接返回,避免重复执行。
|
||||
if run is None or run["status"] not in ("QUEUED", "PAUSED"):
|
||||
return
|
||||
# 已暂停的任务不自动续跑:直接返回保持 PAUSED,等待用户显式 resume
|
||||
# (resume 把状态转回 QUEUED 后才会真正执行)。修复回归——此前以
|
||||
# PAUSED 进入后立即置 RUNNING,节点循环的暂停检查永远不成立,
|
||||
# 任务被复活继续执行("点击暂停反而开始任务")。
|
||||
if run["status"] == "PAUSED":
|
||||
return
|
||||
|
||||
# 工作流或版本记录丢失时把任务标记为失败。
|
||||
workflow = self.db.get_workflow(run["workflow_id"])
|
||||
@@ -143,6 +155,8 @@ class WorkflowScheduler:
|
||||
# 从已登记产物重建已完成节点的输出,支持暂停后断点续跑。
|
||||
outputs_by_node = self.db.restore_run_outputs(run_id)
|
||||
run_started = time.monotonic()
|
||||
# 清除可能残留的暂停信号(重启/异常中断后),避免本次执行误触发节点内暂停。
|
||||
(self.storage_dir / "runs" / run_id / "paused.flag").unlink(missing_ok=True)
|
||||
self.db.update_run(run_id, status="RUNNING", progress=0, updated_at=_now_iso())
|
||||
try:
|
||||
for index, node_id in enumerate(ordered):
|
||||
@@ -249,7 +263,13 @@ class WorkflowScheduler:
|
||||
updated_at=_now_iso(),
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
# 任一步骤异常都结束任务并记录错误,等待用户重试。
|
||||
# 节点执行中被暂停(节点内检测到 paused.flag 而中止):保持 PAUSED
|
||||
# 等待用户 resume 从断点续跑,而不是把暂停误报为 FAILED。
|
||||
current = self.db.get_run(run_id)
|
||||
if current is not None and current["status"] == "PAUSED":
|
||||
logger.info("任务 %s 节点内被暂停,保持 PAUSED: %s", run_id, exc)
|
||||
return
|
||||
# 其余异常:结束任务并记录错误,等待用户重试。
|
||||
self.db.update_run(
|
||||
run_id,
|
||||
status="FAILED",
|
||||
|
||||
Reference in New Issue
Block a user