perf: SQLite 开 WAL 与写锁等待,支持流水线并发写
批量流水线会在多个线程里同时写产物与进度,默认的回滚日志模式与 5 秒 busy timeout 下容易出现 database is locked。 - 连接统一带 `WOV_DB_BUSY_TIMEOUT_SECONDS`(默认 30 秒)等待写锁。 - 打开 WAL 与 `synchronous=NORMAL`:读写并行,落盘开销更低;不支持 WAL 的 文件系统会保持原模式(SQLite 自身降级)。 - 测试:断言 WAL 生效,并用 4 线程 × 25 条并发写产物验证不丢数据、不报错。
This commit is contained in:
@@ -381,3 +381,62 @@ def test_recover_interrupted_batch_jobs_requeues_zombie_completed(db_with_workfl
|
||||
assert count == 1
|
||||
assert db_with_workflow.get_batch_job("zombie")["status"] == "QUEUED"
|
||||
assert db_with_workflow.get_batch_job("done")["status"] == "COMPLETED"
|
||||
|
||||
|
||||
def test_wal_mode_enabled_for_concurrent_pipeline_writes(tmp_path: Path) -> None:
|
||||
"""数据:临时库文件。
|
||||
|
||||
过程:打开一次连接并读取日志模式。
|
||||
|
||||
验证:处于 WAL 模式——批量流水线多线程并发写产物/进度时,读写可并行,
|
||||
不会互相阻塞成 database is locked。
|
||||
"""
|
||||
db = Database(tmp_path / "wov.db")
|
||||
|
||||
with db._connect() as conn:
|
||||
mode = conn.execute("PRAGMA journal_mode").fetchone()[0]
|
||||
|
||||
assert mode.lower() == "wal"
|
||||
|
||||
|
||||
def test_concurrent_artifact_writes_survive(tmp_path: Path) -> None:
|
||||
"""数据:四个线程各自往同一任务写入 25 条产物记录。
|
||||
|
||||
过程:并发调用 create_artifact(批量流水线里多个视频同时落产物的场景)。
|
||||
|
||||
验证:全部写入成功且记录数正确,没有因写锁竞争丢数据或抛异常。
|
||||
"""
|
||||
import threading
|
||||
|
||||
db = Database(tmp_path / "wov.db")
|
||||
db.upsert_workflow({"id": "wf", "name": "流程", "description": "", "published": 1, "latest_version": 1})
|
||||
db.create_workflow_version("wf", 1, {"nodes": []})
|
||||
db.create_run({
|
||||
"id": "run-concurrent", "workflow_id": "wf", "workflow_version": 1,
|
||||
"status": "RUNNING", "source": "batch", "created_at": "t1", "updated_at": "t1",
|
||||
})
|
||||
errors: list[Exception] = []
|
||||
|
||||
def writer(worker: int) -> None:
|
||||
try:
|
||||
for index in range(25):
|
||||
db.create_artifact({
|
||||
"run_id": "run-concurrent", "node_id": "asr",
|
||||
"name": f"w{worker}-{index}", "uri": f"/tmp/w{worker}-{index}.srt",
|
||||
"mime_type": "text/plain", "size": index,
|
||||
})
|
||||
except Exception as exc: # noqa: BLE001 - 用例要报告任意写失败
|
||||
errors.append(exc)
|
||||
|
||||
threads = [threading.Thread(target=writer, args=(n,)) for n in range(4)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
|
||||
assert errors == []
|
||||
with db._connect() as conn:
|
||||
count = conn.execute(
|
||||
"SELECT COUNT(*) FROM artifacts WHERE run_id = ?", ("run-concurrent",),
|
||||
).fetchone()[0]
|
||||
assert count == 100
|
||||
|
||||
Reference in New Issue
Block a user