import time import socket # freejavbt.com 同时解析出 IPv4 和 IPv6 地址。部分网络环境 IPv6 不可达时, # requests 会先尝试 IPv6 连接并长时间无响应(表现为"卡在获取页面信息"步骤)。 # 强制走 IPv4 避免挂起;必须在导入 requests 之前设置。 socket.has_ipv6 = False from loguru import logger from bs4 import BeautifulSoup import cf_session from dataService.dto.SeedInfo import SeedInfo from dataService import MySqlService as mysql from ariaDownloaderService.QbittorrentDownloader import QbittorrentDownloader # 请求超时: (连接超时, 读取超时)。即使 IPv4 也被网络阻断, # 也会在连接超时后抛出异常返回 None,保证程序不会被卡死。 REQUEST_TIMEOUT = (10, 60) """ 获取页面信息 """ def get_page(url): logger.debug("开始获取页面信息:{}", url) # 自动处理 Cloudflare 限流/人机认证:退避重试,连续失败时自动弹浏览器刷新 cookie soup, status = cf_session.fetch_with_retry(url, REQUEST_TIMEOUT) if status == "ok": logger.debug("页面信息获取成功。URL: {}", url) elif status == "rate_limited": logger.warning("页面持续触发 Cloudflare 人机认证/限流,重试及自动刷新后仍未成功。URL: {}。{}", url, cf_session.refresh_hint()) # status == "error" 时 fetch_with_retry 已记录详细错误日志 return soup """ 用于获取目标元素上的详情页url """ def get_target_element_list(page_content): logger.debug("开始提取可下载内容的链接") downloadable_url_list = [] try: for element in page_content.find_all("div", {"class": "card-body"}): if "可下" in element.find("div", {"class": "video-list-item-tag-wrapper"}).text: url = element.find("a")["href"] downloadable_url_list.append(url) logger.debug("发现可下载链接: {}", url) logger.debug("链接提取完成。共发现 {} 个链接。", len(downloadable_url_list)) return downloadable_url_list except Exception as e: logger.error("提取链接时出现错误: {}", str(e)) return [] def get_seed_from_element_detail(element_detail, url=None): logger.debug("开始解析页面中的种子信息") seedInfoList = [] time.sleep(2) try: # 限流或异常页面直接跳过,避免无意义的解析报错 if element_detail is None or "Too Many Requests" in element_detail.text: logger.warning("详情页无效或被限流,跳过解析: {}", url) return [] # 来源页面 URL:优先使用调用方传入的 URL,其次取页面 canonical 链接。 # 注意不能取第一个 ,真实页面首个 link 是 favicon 图标地址。 from_url = url if not from_url: canonical = element_detail.find("link", {"rel": "canonical"}) from_url = canonical["href"] if canonical else None title_element = element_detail.find("meta", {"name": "description"}) if title_element is None: logger.warning("详情页缺少描述信息,跳过解析: {}", url) return [] title = title_element["content"] tbody = element_detail.find("tbody") if tbody is None: logger.warning("详情页缺少种子列表,跳过解析: {}", url) return [] for e in tbody.find_all("a", class_="magnet-name text-truncate d-block small"): seedInfo = SeedInfo(title, e["href"], "1", url=from_url) seedInfoList.append(seedInfo) logger.debug("解析到种子信息: 标题:{},链接:{}", title, e["href"]) logger.debug("页面种子信息解析完成。共 {} 个种子。", len(seedInfoList)) return seedInfoList except Exception as e: logger.error("解析种子信息时出现错误: {}", str(e)) return [] def main(): # 实例化下载器(登录 qBittorrent),供后续下载种子使用 downloader = QbittorrentDownloader() # 1-18 for page in range(5): url = f"https://freejavbt.com/censored/filter?c=212&page={page+1}" logger.info("程序启动。开始处理 URL: {}", url) new_seed = [] # 获取页面信息 page = get_page(url) if page is not None: # 获取页面中的目标元素所包含的详情页信息(url列表) targetElementList = get_target_element_list(page) for targetElement in targetElementList: element_detail = get_page(targetElement) if element_detail is not None: seed_list = get_seed_from_element_detail(element_detail, targetElement) # 将新发现的种子存储进new_seed列表 for seed in seed_list: if not mysql.seed_is_exist(seed): new_seed.append(seed) logger.debug("新增种子到下载列表: {}", seed.name) # 推送new_seed进入aria2进行种子下载 for seed in new_seed: logger.info("开始下载种子: {}", seed.name) status = downloader.download_by_seed(seed) # 返回值取值1,0,-1 seed.haveDownload = status seed.seave_to_dataBase() logger.info("种子下载完成。标题: {}, 状态: {}", seed.name, "成功" if status == "1" else "失败") else: logger.error("无法获取网页内容。URL: {}", url) logger.info("程序处理完成。") if __name__ == "__main__": main()