fix: 修复主程序卡死与下载器集成,新增真实数据测试

- LookingWebMain: 强制 IPv4 避免 IPv6 不可达挂起;requests 加超时防卡死
- get_seed_from_element_detail: 修复限流页崩溃与 favicon 误作来源 URL,改用 canonical/传入 URL
- QbittorrentDownloader: 修复 main() 中模块/类误用导致的 download_by_seed AttributeError;请求加超时
- MySqlService: 修复数据库不可达时 query/execute/close 的 None 崩溃,close 后置空 connection
- 新增 cf_session: Cloudflare 人机认证 cookie 管理 + 限流退避重试 + 自动弹浏览器刷新验证
- 新增 refresh_cf_cookies.py: 弹出浏览器手动验证并保存 cf_clearance
- tests/: 真实网络/数据库测试(get_page、seed_is_exist、cf_session)+ run_tests.py runner
- 清理无用脚本(final_test2/verify_replace/REPLACEMENT_INFO),.idea 与 cookie 文件入 .gitignore

注: test_qbittorrent_downloader.py 11 个用例为既有失败(mock 目标 session 为实例属性,与实现错配),未在本次范围
This commit is contained in:
2026-08-30 12:24:26 +08:00
parent f8e8fd38b4
commit 01827224d6
14 changed files with 1327 additions and 18 deletions
+48 -15
View File
@@ -1,25 +1,35 @@
import time
import socket
# freejavbt.com 同时解析出 IPv4 和 IPv6 地址。部分网络环境 IPv6 不可达时,
# requests 会先尝试 IPv6 连接并长时间无响应(表现为"卡在获取页面信息"步骤)。
# 强制走 IPv4 避免挂起;必须在导入 requests 之前设置。
socket.has_ipv6 = False
import requests
from loguru import logger
from bs4 import BeautifulSoup
import cf_session
from dataService.dto.SeedInfo import SeedInfo
from dataService import MySqlService as mysql
from ariaDownloaderService import AriaDownloader as ariaDownloader
from ariaDownloaderService.QbittorrentDownloader import QbittorrentDownloader
# 请求超时: (连接超时, 读取超时)。即使 IPv4 也被网络阻断,
# 也会在连接超时后抛出异常返回 None,保证程序不会被卡死。
REQUEST_TIMEOUT = (10, 60)
"""
获取页面信息
"""
def get_page(url):
logger.debug("开始获取页面信息:{}", url)
try:
response = requests.get(url)
soup = BeautifulSoup(response.content, 'html.parser')
# 自动处理 Cloudflare 限流/人机认证:退避重试,连续失败时自动弹浏览器刷新 cookie
soup, status = cf_session.fetch_with_retry(url, REQUEST_TIMEOUT)
if status == "ok":
logger.debug("页面信息获取成功。URL: {}", url)
return soup
except Exception as e:
logger.error("获取页面信息失败。URL: {}, 错误: {}", url, str(e))
return None
elif status == "rate_limited":
logger.warning("页面持续触发 Cloudflare 人机认证/限流,重试及自动刷新后仍未成功。URL: {}{}",
url, cf_session.refresh_hint())
# status == "error" 时 fetch_with_retry 已记录详细错误日志
return soup
"""
@@ -41,14 +51,35 @@ def get_target_element_list(page_content):
return []
def get_seed_from_element_detail(element_detail):
def get_seed_from_element_detail(element_detail, url=None):
logger.debug("开始解析页面中的种子信息")
seedInfoList = []
time.sleep(2)
try:
from_url = element_detail.find("link")["href"]
title = element_detail.find("meta", {"name": "description"})["content"]
for e in element_detail.find("tbody").find_all("a", class_="magnet-name text-truncate d-block small"):
# 限流或异常页面直接跳过,避免无意义的解析报错
if element_detail is None or "Too Many Requests" in element_detail.text:
logger.warning("详情页无效或被限流,跳过解析: {}", url)
return []
# 来源页面 URL:优先使用调用方传入的 URL,其次取页面 canonical 链接。
# 注意不能取第一个 <link>,真实页面首个 link 是 favicon 图标地址。
from_url = url
if not from_url:
canonical = element_detail.find("link", {"rel": "canonical"})
from_url = canonical["href"] if canonical else None
title_element = element_detail.find("meta", {"name": "description"})
if title_element is None:
logger.warning("详情页缺少描述信息,跳过解析: {}", url)
return []
title = title_element["content"]
tbody = element_detail.find("tbody")
if tbody is None:
logger.warning("详情页缺少种子列表,跳过解析: {}", url)
return []
for e in tbody.find_all("a", class_="magnet-name text-truncate d-block small"):
seedInfo = SeedInfo(title, e["href"], "1", url=from_url)
seedInfoList.append(seedInfo)
logger.debug("解析到种子信息: 标题:{},链接:{}", title, e["href"])
@@ -60,6 +91,8 @@ def get_seed_from_element_detail(element_detail):
def main():
# 实例化下载器(登录 qBittorrent),供后续下载种子使用
downloader = QbittorrentDownloader()
# 1-18
for page in range(5):
url = f"https://freejavbt.com/censored/filter?c=212&page={page+1}"
@@ -73,7 +106,7 @@ def main():
for targetElement in targetElementList:
element_detail = get_page(targetElement)
if element_detail is not None:
seed_list = get_seed_from_element_detail(element_detail)
seed_list = get_seed_from_element_detail(element_detail, targetElement)
# 将新发现的种子存储进new_seed列表
for seed in seed_list:
if not mysql.seed_is_exist(seed):
@@ -83,7 +116,7 @@ def main():
# 推送new_seed进入aria2进行种子下载
for seed in new_seed:
logger.info("开始下载种子: {}", seed.name)
status = ariaDownloader.download_by_seed(seed) # 返回值取值10-1
status = downloader.download_by_seed(seed) # 返回值取值10-1
seed.haveDownload = status
seed.seave_to_dataBase()
logger.info("种子下载完成。标题: {}, 状态: {}", seed.name, "成功" if status == "1" else "失败")