cat-shark
01827224d6
- LookingWebMain: 强制 IPv4 避免 IPv6 不可达挂起；requests 加超时防卡死 - get_seed_from_element_detail: 修复限流页崩溃与 favicon 误作来源 URL，改用 canonical/传入 URL - QbittorrentDownloader: 修复 main() 中模块/类误用导致的 download_by_seed AttributeError；请求加超时 - MySqlService: 修复数据库不可达时 query/execute/close 的 None 崩溃，close 后置空 connection - 新增 cf_session: Cloudflare 人机认证 cookie 管理 + 限流退避重试 + 自动弹浏览器刷新验证 - 新增 refresh_cf_cookies.py: 弹出浏览器手动验证并保存 cf_clearance - tests/: 真实网络/数据库测试（get_page、seed_is_exist、cf_session）+ run_tests.py runner - 清理无用脚本(final_test2/verify_replace/REPLACEMENT_INFO)，.idea 与 cookie 文件入 .gitignore 注: test_qbittorrent_downloader.py 11 个用例为既有失败（mock 目标 session 为实例属性，与实现错配），未在本次范围
210 lines
7.9 KiB
Python
210 lines
7.9 KiB
Python
"""
|
||
Cloudflare 人机认证 cookie 管理与 requests 会话封装
|
||
|
||
freejavbt.com 在访问频繁时会触发 Cloudflare 人机认证（429 / 403 + JS 挑战页）。
|
||
本模块提供：
|
||
- cf_clearance cookie 的加载 / 保存 / 浏览器自动刷新
|
||
- 带 Chrome User-Agent 的 requests.Session（UA 必须与完成验证的浏览器一致）
|
||
- 页面获取的自动重试：限流退避重试 → 连续失败自动弹出浏览器重新验证
|
||
"""
|
||
import json
|
||
import os
|
||
import socket
|
||
import subprocess
|
||
import time
|
||
|
||
# freejavbt.com 同时解析出 IPv4 和 IPv6，本环境 IPv6 不可达时 requests 会挂起，
|
||
# 强制走 IPv4（必须在导入 requests 之前设置）
|
||
socket.has_ipv6 = False
|
||
|
||
import requests
|
||
from bs4 import BeautifulSoup
|
||
from loguru import logger
|
||
|
||
# cookie 存储文件（与 refresh_cf_cookies.py 约定一致）
|
||
COOKIE_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'cf_cookies.json')
|
||
|
||
# 必须与完成验证的浏览器 UA 一致，cf_clearance 与 UA 绑定
|
||
CHROME_UA = ('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
|
||
'(KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36')
|
||
|
||
# 浏览器验证时打开的站点入口
|
||
SITE_URL = "https://freejavbt.com/censored/filter?c=212&page=1"
|
||
|
||
# Cloudflare 限流/人机认证页的识别标记：
|
||
# - "Too Many Requests" → 429 限流页
|
||
# - "Just a moment" → 403 JS 挑战页（interstitial）
|
||
# - "Attention Required" → 403 阻止页
|
||
RATE_LIMIT_MARKS = ("Too Many Requests", "Just a moment", "Attention Required")
|
||
|
||
_session = None
|
||
# 每次进程内只自动弹浏览器刷新一次，避免大面积限流时反复弹窗
|
||
_refresh_attempted = False
|
||
|
||
|
||
def load_cf_cookies():
|
||
"""从 cf_cookies.json 加载 cookie 字典；文件不存在或损坏时返回 {}"""
|
||
if not os.path.exists(COOKIE_FILE):
|
||
return {}
|
||
try:
|
||
with open(COOKIE_FILE, 'r', encoding='utf-8') as f:
|
||
data = json.load(f)
|
||
return data if isinstance(data, dict) else {}
|
||
except (json.JSONDecodeError, OSError):
|
||
return {}
|
||
|
||
|
||
def save_cf_cookies(cookies):
|
||
"""保存 cookie 字典到 cf_cookies.json"""
|
||
with open(COOKIE_FILE, 'w', encoding='utf-8') as f:
|
||
json.dump(cookies, f, ensure_ascii=False, indent=2)
|
||
|
||
|
||
def reset_session():
|
||
"""清空缓存会话，下次 get_session() 会从 cf_cookies.json 重建"""
|
||
global _session
|
||
_session = None
|
||
|
||
|
||
def get_session():
|
||
"""构造（或复用）带 cf_clearance + Chrome UA 的 requests.Session"""
|
||
global _session
|
||
if _session is None:
|
||
_session = requests.Session()
|
||
_session.headers.update({
|
||
'User-Agent': CHROME_UA,
|
||
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
|
||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||
})
|
||
for name, value in load_cf_cookies().items():
|
||
_session.cookies.set(name, value, domain='.freejavbt.com', path='/')
|
||
return _session
|
||
|
||
|
||
def is_rate_limited(page_content):
|
||
"""判断页面是否为 Cloudflare 限流/人机认证页（429 与 403 两种形态）"""
|
||
if page_content is None:
|
||
return False
|
||
text = page_content.text
|
||
return any(mark in text for mark in RATE_LIMIT_MARKS)
|
||
|
||
|
||
def refresh_hint():
|
||
"""cookie 失效时提示用户重新验证"""
|
||
return ("检测到 Cloudflare 人机认证/限流，cf_clearance cookie 可能已过期。"
|
||
"请运行: python refresh_cf_cookies.py，在弹出的浏览器中完成验证后重试")
|
||
|
||
|
||
def _parse_cookies(stdout):
|
||
"""解析 'name=value' 行列表为 cookie 字典"""
|
||
cookies = {}
|
||
for line in stdout.splitlines():
|
||
line = line.strip()
|
||
if '=' in line and not line.startswith('#'):
|
||
name, _, value = line.partition('=')
|
||
cookies[name.strip()] = value.strip()
|
||
return cookies
|
||
|
||
|
||
def refresh_cookies(max_wait=120):
|
||
"""
|
||
弹出真实浏览器（headed）完成 Cloudflare 人机验证，自动保存新 cookie。
|
||
返回 True 表示验证成功并已刷新 cookie。
|
||
"""
|
||
global _refresh_attempted, _session
|
||
_refresh_attempted = True
|
||
logger.warning("自动弹出浏览器完成 Cloudflare 人机验证...")
|
||
|
||
# 关闭旧会话，确保可以带窗口启动
|
||
try:
|
||
subprocess.run(["agent-browser", "close"], capture_output=True, timeout=60)
|
||
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||
pass
|
||
|
||
# 弹出带窗口的真实浏览器
|
||
try:
|
||
proc = subprocess.run(
|
||
["agent-browser", "open", SITE_URL, "--headed", "--timeout", "60000"],
|
||
capture_output=True, text=True, timeout=90)
|
||
if proc.returncode != 0:
|
||
logger.error("浏览器启动失败: {}", proc.stderr.strip()[:200])
|
||
return False
|
||
except (subprocess.TimeoutExpired, FileNotFoundError) as e:
|
||
logger.error("无法启动浏览器刷新 cookie: {}", e)
|
||
return False
|
||
|
||
# 轮询等待 cf_clearance
|
||
deadline = time.time() + max_wait
|
||
while time.time() < deadline:
|
||
time.sleep(3)
|
||
try:
|
||
proc = subprocess.run(["agent-browser", "cookies"],
|
||
capture_output=True, text=True, timeout=30)
|
||
except (subprocess.TimeoutExpired, FileNotFoundError) as e:
|
||
logger.error("读取 cookie 失败: {}", e)
|
||
return False
|
||
if proc.returncode != 0:
|
||
continue
|
||
cookies = _parse_cookies(proc.stdout)
|
||
if "cf_clearance" in cookies:
|
||
save_cf_cookies(cookies)
|
||
reset_session()
|
||
logger.info("人机验证完成，cookie 已刷新到 {}", COOKIE_FILE)
|
||
return True
|
||
|
||
logger.error("{} 秒内未检测到 cf_clearance，验证超时", max_wait)
|
||
return False
|
||
|
||
|
||
def fetch_with_retry(url, timeout, retries=3, retry_delay=5, auto_refresh=True):
|
||
"""
|
||
获取页面并自动处理 Cloudflare 限流/人机认证：
|
||
1. 触发挑战 → 等待 retry_delay 秒退避重试，最多 retries 次
|
||
2. 重试耗尽仍被挑战 → 自动弹出浏览器重新验证（每个进程仅一次）→ 再试 1 次
|
||
返回 (soup, status)：
|
||
- status='ok' 正常页面
|
||
- status='rate_limited' 重试/刷新后仍被挑战（soup 为挑战页内容）
|
||
- status='error' 网络异常（soup 为 None）
|
||
"""
|
||
global _refresh_attempted
|
||
session = get_session()
|
||
last_soup = None
|
||
status = "error"
|
||
|
||
for attempt in range(1, retries + 1):
|
||
try:
|
||
response = session.get(url, timeout=timeout)
|
||
last_soup = BeautifulSoup(response.content, "html.parser")
|
||
if is_rate_limited(last_soup):
|
||
status = "rate_limited"
|
||
logger.warning("触发 Cloudflare 人机认证/限流（第 {}/{} 次），{} 秒后重试: {}",
|
||
attempt, retries, retry_delay, url)
|
||
if attempt < retries:
|
||
time.sleep(retry_delay)
|
||
continue
|
||
return last_soup, "ok"
|
||
except Exception as e:
|
||
status = "error"
|
||
last_soup = None
|
||
logger.error("请求失败（第 {}/{} 次）: {}，错误: {}", attempt, retries, url, e)
|
||
if attempt < retries:
|
||
time.sleep(retry_delay)
|
||
|
||
# 限流重试耗尽 → 自动刷新 cookie 后再试一次
|
||
if status == "rate_limited" and auto_refresh and not _refresh_attempted:
|
||
if refresh_cookies():
|
||
try:
|
||
response = session.get(url, timeout=timeout)
|
||
last_soup = BeautifulSoup(response.content, "html.parser")
|
||
if not is_rate_limited(last_soup):
|
||
logger.info("刷新 cookie 后获取成功: {}", url)
|
||
return last_soup, "ok"
|
||
status = "rate_limited"
|
||
logger.warning("刷新 cookie 后仍被限流: {}", url)
|
||
except Exception as e:
|
||
status = "error"
|
||
last_soup = None
|
||
logger.error("刷新 cookie 后请求失败: {}，错误: {}", url, e)
|
||
|
||
return last_soup, status
|