97 lines
3.9 KiB
Python
97 lines
3.9 KiB
Python
import time
|
||
|
||
import requests
|
||
from loguru import logger
|
||
from bs4 import BeautifulSoup
|
||
from dataService.dto.SeedInfo import SeedInfo
|
||
from dataService import MySqlService as mysql
|
||
from ariaDownloaderService import AriaDownloader as ariaDownloader
|
||
|
||
"""
|
||
获取页面信息
|
||
"""
|
||
def get_page(url):
|
||
logger.debug("开始获取页面信息：{}", url)
|
||
try:
|
||
response = requests.get(url)
|
||
soup = BeautifulSoup(response.content, 'html.parser')
|
||
logger.debug("页面信息获取成功。URL: {}", url)
|
||
return soup
|
||
except Exception as e:
|
||
logger.error("获取页面信息失败。URL: {}, 错误: {}", url, str(e))
|
||
return None
|
||
|
||
|
||
"""
|
||
用于获取目标元素上的详情页url
|
||
"""
|
||
def get_target_element_list(page_content):
|
||
logger.debug("开始提取可下载内容的链接")
|
||
downloadable_url_list = []
|
||
try:
|
||
for element in page_content.find_all("div", {"class": "card-body"}):
|
||
if "可下" in element.find("div", {"class": "video-list-item-tag-wrapper"}).text:
|
||
url = element.find("a")["href"]
|
||
downloadable_url_list.append(url)
|
||
logger.debug("发现可下载链接: {}", url)
|
||
logger.debug("链接提取完成。共发现 {} 个链接。", len(downloadable_url_list))
|
||
return downloadable_url_list
|
||
except Exception as e:
|
||
logger.error("提取链接时出现错误: {}", str(e))
|
||
return []
|
||
|
||
|
||
def get_seed_from_element_detail(element_detail):
|
||
logger.debug("开始解析页面中的种子信息")
|
||
seedInfoList = []
|
||
time.sleep(2)
|
||
try:
|
||
from_url = element_detail.find("link")["href"]
|
||
title = element_detail.find("meta", {"name": "description"})["content"]
|
||
for e in element_detail.find("tbody").find_all("a", class_="magnet-name text-truncate d-block small"):
|
||
seedInfo = SeedInfo(title, e["href"], "1", url=from_url)
|
||
seedInfoList.append(seedInfo)
|
||
logger.debug("解析到种子信息: 标题：{}，链接：{}", title, e["href"])
|
||
logger.debug("页面种子信息解析完成。共 {} 个种子。", len(seedInfoList))
|
||
return seedInfoList
|
||
except Exception as e:
|
||
logger.error("解析种子信息时出现错误: {}", str(e))
|
||
return []
|
||
|
||
|
||
def main():
|
||
# 1-18
|
||
for page in range(5):
|
||
url = f"https://freejavbt.com/censored/filter?c=212&page={page+1}"
|
||
logger.info("程序启动。开始处理 URL: {}", url)
|
||
new_seed = []
|
||
# 获取页面信息
|
||
page = get_page(url)
|
||
if page is not None:
|
||
# 获取页面中的目标元素所包含的详情页信息(url列表)
|
||
targetElementList = get_target_element_list(page)
|
||
for targetElement in targetElementList:
|
||
element_detail = get_page(targetElement)
|
||
if element_detail is not None:
|
||
seed_list = get_seed_from_element_detail(element_detail)
|
||
# 将新发现的种子存储进new_seed列表
|
||
for seed in seed_list:
|
||
if not mysql.seed_is_exist(seed):
|
||
new_seed.append(seed)
|
||
logger.debug("新增种子到下载列表: {}", seed.name)
|
||
|
||
# 推送new_seed进入aria2进行种子下载
|
||
for seed in new_seed:
|
||
logger.info("开始下载种子: {}", seed.name)
|
||
status = ariaDownloader.download_by_seed(seed) # 返回值取值1，0，-1
|
||
seed.haveDownload = status
|
||
seed.seave_to_dataBase()
|
||
logger.info("种子下载完成。标题: {}, 状态: {}", seed.name, "成功" if status == "1" else "失败")
|
||
else:
|
||
logger.error("无法获取网页内容。URL: {}", url)
|
||
|
||
logger.info("程序处理完成。")
|
||
|
||
if __name__ == "__main__":
|
||
main()
|