cat-shark
01827224d6
- LookingWebMain: 强制 IPv4 避免 IPv6 不可达挂起；requests 加超时防卡死 - get_seed_from_element_detail: 修复限流页崩溃与 favicon 误作来源 URL，改用 canonical/传入 URL - QbittorrentDownloader: 修复 main() 中模块/类误用导致的 download_by_seed AttributeError；请求加超时 - MySqlService: 修复数据库不可达时 query/execute/close 的 None 崩溃，close 后置空 connection - 新增 cf_session: Cloudflare 人机认证 cookie 管理 + 限流退避重试 + 自动弹浏览器刷新验证 - 新增 refresh_cf_cookies.py: 弹出浏览器手动验证并保存 cf_clearance - tests/: 真实网络/数据库测试（get_page、seed_is_exist、cf_session）+ run_tests.py runner - 清理无用脚本(final_test2/verify_replace/REPLACEMENT_INFO)，.idea 与 cookie 文件入 .gitignore 注: test_qbittorrent_downloader.py 11 个用例为既有失败（mock 目标 session 为实例属性，与实现错配），未在本次范围
130 lines
5.7 KiB
Python
130 lines
5.7 KiB
Python
import time
|
||
import socket
|
||
|
||
# freejavbt.com 同时解析出 IPv4 和 IPv6 地址。部分网络环境 IPv6 不可达时，
|
||
# requests 会先尝试 IPv6 连接并长时间无响应（表现为"卡在获取页面信息"步骤）。
|
||
# 强制走 IPv4 避免挂起；必须在导入 requests 之前设置。
|
||
socket.has_ipv6 = False
|
||
|
||
from loguru import logger
|
||
from bs4 import BeautifulSoup
|
||
import cf_session
|
||
from dataService.dto.SeedInfo import SeedInfo
|
||
from dataService import MySqlService as mysql
|
||
from ariaDownloaderService.QbittorrentDownloader import QbittorrentDownloader
|
||
# 请求超时: (连接超时, 读取超时)。即使 IPv4 也被网络阻断，
|
||
# 也会在连接超时后抛出异常返回 None，保证程序不会被卡死。
|
||
REQUEST_TIMEOUT = (10, 60)
|
||
|
||
"""
|
||
获取页面信息
|
||
"""
|
||
def get_page(url):
|
||
logger.debug("开始获取页面信息：{}", url)
|
||
# 自动处理 Cloudflare 限流/人机认证：退避重试，连续失败时自动弹浏览器刷新 cookie
|
||
soup, status = cf_session.fetch_with_retry(url, REQUEST_TIMEOUT)
|
||
if status == "ok":
|
||
logger.debug("页面信息获取成功。URL: {}", url)
|
||
elif status == "rate_limited":
|
||
logger.warning("页面持续触发 Cloudflare 人机认证/限流，重试及自动刷新后仍未成功。URL: {}。{}",
|
||
url, cf_session.refresh_hint())
|
||
# status == "error" 时 fetch_with_retry 已记录详细错误日志
|
||
return soup
|
||
|
||
|
||
"""
|
||
用于获取目标元素上的详情页url
|
||
"""
|
||
def get_target_element_list(page_content):
|
||
logger.debug("开始提取可下载内容的链接")
|
||
downloadable_url_list = []
|
||
try:
|
||
for element in page_content.find_all("div", {"class": "card-body"}):
|
||
if "可下" in element.find("div", {"class": "video-list-item-tag-wrapper"}).text:
|
||
url = element.find("a")["href"]
|
||
downloadable_url_list.append(url)
|
||
logger.debug("发现可下载链接: {}", url)
|
||
logger.debug("链接提取完成。共发现 {} 个链接。", len(downloadable_url_list))
|
||
return downloadable_url_list
|
||
except Exception as e:
|
||
logger.error("提取链接时出现错误: {}", str(e))
|
||
return []
|
||
|
||
|
||
def get_seed_from_element_detail(element_detail, url=None):
|
||
logger.debug("开始解析页面中的种子信息")
|
||
seedInfoList = []
|
||
time.sleep(2)
|
||
try:
|
||
# 限流或异常页面直接跳过，避免无意义的解析报错
|
||
if element_detail is None or "Too Many Requests" in element_detail.text:
|
||
logger.warning("详情页无效或被限流，跳过解析: {}", url)
|
||
return []
|
||
|
||
# 来源页面 URL：优先使用调用方传入的 URL，其次取页面 canonical 链接。
|
||
# 注意不能取第一个 <link>，真实页面首个 link 是 favicon 图标地址。
|
||
from_url = url
|
||
if not from_url:
|
||
canonical = element_detail.find("link", {"rel": "canonical"})
|
||
from_url = canonical["href"] if canonical else None
|
||
|
||
title_element = element_detail.find("meta", {"name": "description"})
|
||
if title_element is None:
|
||
logger.warning("详情页缺少描述信息，跳过解析: {}", url)
|
||
return []
|
||
title = title_element["content"]
|
||
|
||
tbody = element_detail.find("tbody")
|
||
if tbody is None:
|
||
logger.warning("详情页缺少种子列表，跳过解析: {}", url)
|
||
return []
|
||
|
||
for e in tbody.find_all("a", class_="magnet-name text-truncate d-block small"):
|
||
seedInfo = SeedInfo(title, e["href"], "1", url=from_url)
|
||
seedInfoList.append(seedInfo)
|
||
logger.debug("解析到种子信息: 标题：{}，链接：{}", title, e["href"])
|
||
logger.debug("页面种子信息解析完成。共 {} 个种子。", len(seedInfoList))
|
||
return seedInfoList
|
||
except Exception as e:
|
||
logger.error("解析种子信息时出现错误: {}", str(e))
|
||
return []
|
||
|
||
|
||
def main():
|
||
# 实例化下载器（登录 qBittorrent），供后续下载种子使用
|
||
downloader = QbittorrentDownloader()
|
||
# 1-18
|
||
for page in range(5):
|
||
url = f"https://freejavbt.com/censored/filter?c=212&page={page+1}"
|
||
logger.info("程序启动。开始处理 URL: {}", url)
|
||
new_seed = []
|
||
# 获取页面信息
|
||
page = get_page(url)
|
||
if page is not None:
|
||
# 获取页面中的目标元素所包含的详情页信息(url列表)
|
||
targetElementList = get_target_element_list(page)
|
||
for targetElement in targetElementList:
|
||
element_detail = get_page(targetElement)
|
||
if element_detail is not None:
|
||
seed_list = get_seed_from_element_detail(element_detail, targetElement)
|
||
# 将新发现的种子存储进new_seed列表
|
||
for seed in seed_list:
|
||
if not mysql.seed_is_exist(seed):
|
||
new_seed.append(seed)
|
||
logger.debug("新增种子到下载列表: {}", seed.name)
|
||
|
||
# 推送new_seed进入aria2进行种子下载
|
||
for seed in new_seed:
|
||
logger.info("开始下载种子: {}", seed.name)
|
||
status = downloader.download_by_seed(seed) # 返回值取值1，0，-1
|
||
seed.haveDownload = status
|
||
seed.seave_to_dataBase()
|
||
logger.info("种子下载完成。标题: {}, 状态: {}", seed.name, "成功" if status == "1" else "失败")
|
||
else:
|
||
logger.error("无法获取网页内容。URL: {}", url)
|
||
|
||
logger.info("程序处理完成。")
|
||
|
||
if __name__ == "__main__":
|
||
main()
|