fix: 修复主程序卡死与下载器集成，新增真实数据测试

- LookingWebMain: 强制 IPv4 避免 IPv6 不可达挂起；requests 加超时防卡死 - get_seed_from_element_detail: 修复限流页崩溃与 favicon 误作来源 URL，改用 canonical/传入 URL - QbittorrentDownloader: 修复 main() 中模块/类误用导致的 download_by_seed AttributeError；请求加超时 - MySqlService: 修复数据库不可达时 query/execute/close 的 None 崩溃，close 后置空 connection - 新增 cf_session: Cloudflare 人机认证 cookie 管理 + 限流退避重试 + 自动弹浏览器刷新验证 - 新增 refresh_cf_cookies.py: 弹出浏览器手动验证并保存 cf_clearance - tests/: 真实网络/数据库测试（get_page、seed_is_exist、cf_session）+ run_tests.py runner - 清理无用脚本(final_test2/verify_replace/REPLACEMENT_INFO)，.idea 与 cookie 文件入 .gitignore 注: test_qbittorrent_downloader.py 11 个用例为既有失败（mock 目标 session 为实例属性，与实现错配），未在本次范围
2026-08-30 12:24:26 +08:00
parent f8e8fd38b4
commit 01827224d6
14 changed files with 1327 additions and 18 deletions
@@ -0,0 +1,209 @@
+"""
+Cloudflare 人机认证 cookie 管理与 requests 会话封装
+
+freejavbt.com 在访问频繁时会触发 Cloudflare 人机认证（429 / 403 + JS 挑战页）。
+本模块提供：
+- cf_clearance cookie 的加载 / 保存 / 浏览器自动刷新
+- 带 Chrome User-Agent 的 requests.Session（UA 必须与完成验证的浏览器一致）
+- 页面获取的自动重试：限流退避重试 → 连续失败自动弹出浏览器重新验证
+"""
+import json
+import os
+import socket
+import subprocess
+import time
+
+# freejavbt.com 同时解析出 IPv4 和 IPv6，本环境 IPv6 不可达时 requests 会挂起，
+# 强制走 IPv4（必须在导入 requests 之前设置）
+socket.has_ipv6 = False
+
+import requests
+from bs4 import BeautifulSoup
+from loguru import logger
+
+# cookie 存储文件（与 refresh_cf_cookies.py 约定一致）
+COOKIE_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'cf_cookies.json')
+
+# 必须与完成验证的浏览器 UA 一致，cf_clearance 与 UA 绑定
+CHROME_UA = ('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
+             '(KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36')
+
+# 浏览器验证时打开的站点入口
+SITE_URL = "https://freejavbt.com/censored/filter?c=212&page=1"
+
+# Cloudflare 限流/人机认证页的识别标记：
+# - "Too Many Requests"      → 429 限流页
+# - "Just a moment"          → 403 JS 挑战页（interstitial）
+# - "Attention Required"     → 403 阻止页
+RATE_LIMIT_MARKS = ("Too Many Requests", "Just a moment", "Attention Required")
+
+_session = None
+# 每次进程内只自动弹浏览器刷新一次，避免大面积限流时反复弹窗
+_refresh_attempted = False
+
+
+def load_cf_cookies():
+    """从 cf_cookies.json 加载 cookie 字典；文件不存在或损坏时返回 {}"""
+    if not os.path.exists(COOKIE_FILE):
+        return {}
+    try:
+        with open(COOKIE_FILE, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+            return data if isinstance(data, dict) else {}
+    except (json.JSONDecodeError, OSError):
+        return {}
+
+
+def save_cf_cookies(cookies):
+    """保存 cookie 字典到 cf_cookies.json"""
+    with open(COOKIE_FILE, 'w', encoding='utf-8') as f:
+        json.dump(cookies, f, ensure_ascii=False, indent=2)
+
+
+def reset_session():
+    """清空缓存会话，下次 get_session() 会从 cf_cookies.json 重建"""
+    global _session
+    _session = None
+
+
+def get_session():
+    """构造（或复用）带 cf_clearance + Chrome UA 的 requests.Session"""
+    global _session
+    if _session is None:
+        _session = requests.Session()
+        _session.headers.update({
+            'User-Agent': CHROME_UA,
+            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        })
+        for name, value in load_cf_cookies().items():
+            _session.cookies.set(name, value, domain='.freejavbt.com', path='/')
+    return _session
+
+
+def is_rate_limited(page_content):
+    """判断页面是否为 Cloudflare 限流/人机认证页（429 与 403 两种形态）"""
+    if page_content is None:
+        return False
+    text = page_content.text
+    return any(mark in text for mark in RATE_LIMIT_MARKS)
+
+
+def refresh_hint():
+    """cookie 失效时提示用户重新验证"""
+    return ("检测到 Cloudflare 人机认证/限流，cf_clearance cookie 可能已过期。"
+            "请运行: python refresh_cf_cookies.py，在弹出的浏览器中完成验证后重试")
+
+
+def _parse_cookies(stdout):
+    """解析 'name=value' 行列表为 cookie 字典"""
+    cookies = {}
+    for line in stdout.splitlines():
+        line = line.strip()
+        if '=' in line and not line.startswith('#'):
+            name, _, value = line.partition('=')
+            cookies[name.strip()] = value.strip()
+    return cookies
+
+
+def refresh_cookies(max_wait=120):
+    """
+    弹出真实浏览器（headed）完成 Cloudflare 人机验证，自动保存新 cookie。
+    返回 True 表示验证成功并已刷新 cookie。
+    """
+    global _refresh_attempted, _session
+    _refresh_attempted = True
+    logger.warning("自动弹出浏览器完成 Cloudflare 人机验证...")
+
+    # 关闭旧会话，确保可以带窗口启动
+    try:
+        subprocess.run(["agent-browser", "close"], capture_output=True, timeout=60)
+    except (subprocess.TimeoutExpired, FileNotFoundError):
+        pass
+
+    # 弹出带窗口的真实浏览器
+    try:
+        proc = subprocess.run(
+            ["agent-browser", "open", SITE_URL, "--headed", "--timeout", "60000"],
+            capture_output=True, text=True, timeout=90)
+        if proc.returncode != 0:
+            logger.error("浏览器启动失败: {}", proc.stderr.strip()[:200])
+            return False
+    except (subprocess.TimeoutExpired, FileNotFoundError) as e:
+        logger.error("无法启动浏览器刷新 cookie: {}", e)
+        return False
+
+    # 轮询等待 cf_clearance
+    deadline = time.time() + max_wait
+    while time.time() < deadline:
+        time.sleep(3)
+        try:
+            proc = subprocess.run(["agent-browser", "cookies"],
+                                  capture_output=True, text=True, timeout=30)
+        except (subprocess.TimeoutExpired, FileNotFoundError) as e:
+            logger.error("读取 cookie 失败: {}", e)
+            return False
+        if proc.returncode != 0:
+            continue
+        cookies = _parse_cookies(proc.stdout)
+        if "cf_clearance" in cookies:
+            save_cf_cookies(cookies)
+            reset_session()
+            logger.info("人机验证完成，cookie 已刷新到 {}", COOKIE_FILE)
+            return True
+
+    logger.error("{} 秒内未检测到 cf_clearance，验证超时", max_wait)
+    return False
+
+
+def fetch_with_retry(url, timeout, retries=3, retry_delay=5, auto_refresh=True):
+    """
+    获取页面并自动处理 Cloudflare 限流/人机认证：
+    1. 触发挑战 → 等待 retry_delay 秒退避重试，最多 retries 次
+    2. 重试耗尽仍被挑战 → 自动弹出浏览器重新验证（每个进程仅一次）→ 再试 1 次
+    返回 (soup, status)：
+    - status='ok'           正常页面
+    - status='rate_limited' 重试/刷新后仍被挑战（soup 为挑战页内容）
+    - status='error'        网络异常（soup 为 None）
+    """
+    global _refresh_attempted
+    session = get_session()
+    last_soup = None
+    status = "error"
+
+    for attempt in range(1, retries + 1):
+        try:
+            response = session.get(url, timeout=timeout)
+            last_soup = BeautifulSoup(response.content, "html.parser")
+            if is_rate_limited(last_soup):
+                status = "rate_limited"
+                logger.warning("触发 Cloudflare 人机认证/限流（第 {}/{} 次），{} 秒后重试: {}",
+                               attempt, retries, retry_delay, url)
+                if attempt < retries:
+                    time.sleep(retry_delay)
+                continue
+            return last_soup, "ok"
+        except Exception as e:
+            status = "error"
+            last_soup = None
+            logger.error("请求失败（第 {}/{} 次）: {}，错误: {}", attempt, retries, url, e)
+            if attempt < retries:
+                time.sleep(retry_delay)
+
+    # 限流重试耗尽 → 自动刷新 cookie 后再试一次
+    if status == "rate_limited" and auto_refresh and not _refresh_attempted:
+        if refresh_cookies():
+            try:
+                response = session.get(url, timeout=timeout)
+                last_soup = BeautifulSoup(response.content, "html.parser")
+                if not is_rate_limited(last_soup):
+                    logger.info("刷新 cookie 后获取成功: {}", url)
+                    return last_soup, "ok"
+                status = "rate_limited"
+                logger.warning("刷新 cookie 后仍被限流: {}", url)
+            except Exception as e:
+                status = "error"
+                last_soup = None
+                logger.error("刷新 cookie 后请求失败: {}，错误: {}", url, e)
+
+    return last_soup, status