""" Cloudflare 人机认证 cookie 管理与 requests 会话封装 freejavbt.com 在访问频繁时会触发 Cloudflare 人机认证（429 / 403 + JS 挑战页）。 本模块提供： - cf_clearance cookie 的加载 / 保存 / 浏览器自动刷新 - 带 Chrome User-Agent 的 requests.Session（UA 必须与完成验证的浏览器一致） - 页面获取的自动重试：限流退避重试 → 连续失败自动弹出浏览器重新验证 """ import json import os import socket import subprocess import time # freejavbt.com 同时解析出 IPv4 和 IPv6，本环境 IPv6 不可达时 requests 会挂起， # 强制走 IPv4（必须在导入 requests 之前设置） socket.has_ipv6 = False import requests from bs4 import BeautifulSoup from loguru import logger # cookie 存储文件（与 refresh_cf_cookies.py 约定一致） COOKIE_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'cf_cookies.json') # 必须与完成验证的浏览器 UA 一致，cf_clearance 与 UA 绑定 CHROME_UA = ('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' '(KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36') # 浏览器验证时打开的站点入口 SITE_URL = "https://freejavbt.com/censored/filter?c=212&page=1" # Cloudflare 限流/人机认证页的识别标记： # - "Too Many Requests" → 429 限流页 # - "Just a moment" → 403 JS 挑战页（interstitial） # - "Attention Required" → 403 阻止页 RATE_LIMIT_MARKS = ("Too Many Requests", "Just a moment", "Attention Required") _session = None # 每次进程内只自动弹浏览器刷新一次，避免大面积限流时反复弹窗 _refresh_attempted = False def load_cf_cookies(): """从 cf_cookies.json 加载 cookie 字典；文件不存在或损坏时返回 {}""" if not os.path.exists(COOKIE_FILE): return {} try: with open(COOKIE_FILE, 'r', encoding='utf-8') as f: data = json.load(f) return data if isinstance(data, dict) else {} except (json.JSONDecodeError, OSError): return {} def save_cf_cookies(cookies): """保存 cookie 字典到 cf_cookies.json""" with open(COOKIE_FILE, 'w', encoding='utf-8') as f: json.dump(cookies, f, ensure_ascii=False, indent=2) def reset_session(): """清空缓存会话，下次 get_session() 会从 cf_cookies.json 重建""" global _session _session = None def get_session(): """构造（或复用）带 cf_clearance + Chrome UA 的 requests.Session""" global _session if _session is None: _session = requests.Session() _session.headers.update({ 'User-Agent': CHROME_UA, 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', }) for name, value in load_cf_cookies().items(): _session.cookies.set(name, value, domain='.freejavbt.com', path='/') return _session def is_rate_limited(page_content): """判断页面是否为 Cloudflare 限流/人机认证页（429 与 403 两种形态）""" if page_content is None: return False text = page_content.text return any(mark in text for mark in RATE_LIMIT_MARKS) def refresh_hint(): """cookie 失效时提示用户重新验证""" return ("检测到 Cloudflare 人机认证/限流，cf_clearance cookie 可能已过期。" "请运行: python refresh_cf_cookies.py，在弹出的浏览器中完成验证后重试") def _parse_cookies(stdout): """解析 'name=value' 行列表为 cookie 字典""" cookies = {} for line in stdout.splitlines(): line = line.strip() if '=' in line and not line.startswith('#'): name, _, value = line.partition('=') cookies[name.strip()] = value.strip() return cookies def refresh_cookies(max_wait=120): """ 弹出真实浏览器（headed）完成 Cloudflare 人机验证，自动保存新 cookie。 返回 True 表示验证成功并已刷新 cookie。 """ global _refresh_attempted, _session _refresh_attempted = True logger.warning("自动弹出浏览器完成 Cloudflare 人机验证...") # 关闭旧会话，确保可以带窗口启动 try: subprocess.run(["agent-browser", "close"], capture_output=True, timeout=60) except (subprocess.TimeoutExpired, FileNotFoundError): pass # 弹出带窗口的真实浏览器 try: proc = subprocess.run( ["agent-browser", "open", SITE_URL, "--headed", "--timeout", "60000"], capture_output=True, text=True, timeout=90) if proc.returncode != 0: logger.error("浏览器启动失败: {}", proc.stderr.strip()[:200]) return False except (subprocess.TimeoutExpired, FileNotFoundError) as e: logger.error("无法启动浏览器刷新 cookie: {}", e) return False # 轮询等待 cf_clearance deadline = time.time() + max_wait while time.time() < deadline: time.sleep(3) try: proc = subprocess.run(["agent-browser", "cookies"], capture_output=True, text=True, timeout=30) except (subprocess.TimeoutExpired, FileNotFoundError) as e: logger.error("读取 cookie 失败: {}", e) return False if proc.returncode != 0: continue cookies = _parse_cookies(proc.stdout) if "cf_clearance" in cookies: save_cf_cookies(cookies) reset_session() logger.info("人机验证完成，cookie 已刷新到 {}", COOKIE_FILE) return True logger.error("{} 秒内未检测到 cf_clearance，验证超时", max_wait) return False def fetch_with_retry(url, timeout, retries=3, retry_delay=5, auto_refresh=True): """ 获取页面并自动处理 Cloudflare 限流/人机认证： 1. 触发挑战 → 等待 retry_delay 秒退避重试，最多 retries 次 2. 重试耗尽仍被挑战 → 自动弹出浏览器重新验证（每个进程仅一次）→ 再试 1 次 返回 (soup, status)： - status='ok' 正常页面 - status='rate_limited' 重试/刷新后仍被挑战（soup 为挑战页内容） - status='error' 网络异常（soup 为 None） """ global _refresh_attempted session = get_session() last_soup = None status = "error" for attempt in range(1, retries + 1): try: response = session.get(url, timeout=timeout) last_soup = BeautifulSoup(response.content, "html.parser") if is_rate_limited(last_soup): status = "rate_limited" logger.warning("触发 Cloudflare 人机认证/限流（第 {}/{} 次），{} 秒后重试: {}", attempt, retries, retry_delay, url) if attempt < retries: time.sleep(retry_delay) continue return last_soup, "ok" except Exception as e: status = "error" last_soup = None logger.error("请求失败（第 {}/{} 次）: {}，错误: {}", attempt, retries, url, e) if attempt < retries: time.sleep(retry_delay) # 限流重试耗尽 → 自动刷新 cookie 后再试一次 if status == "rate_limited" and auto_refresh and not _refresh_attempted: if refresh_cookies(): try: response = session.get(url, timeout=timeout) last_soup = BeautifulSoup(response.content, "html.parser") if not is_rate_limited(last_soup): logger.info("刷新 cookie 后获取成功: {}", url) return last_soup, "ok" status = "rate_limited" logger.warning("刷新 cookie 后仍被限流: {}", url) except Exception as e: status = "error" last_soup = None logger.error("刷新 cookie 后请求失败: {}，错误: {}", url, e) return last_soup, status