Files
lpt-be/src/main/java/com/guo/learningprogresstracker/utils/TitleFetcher.java
T
cat_shark 53a6781092 feat: 新增 URL 标题抓取工具
- UtilsController 提供 /utils/fetch-title 接口,供前端获取网页标题
- TitleFetcher 工具类支持 HTTP/HTTPS 请求、重定向跟踪、gzip/deflate 解压
- 支持 SSL 宽松配置、编码自动检测、批量并行抓取
- 支持裸 URL 自动嵌入为 [标题](url) Markdown 格式
2026-07-12 13:43:58 +08:00

274 lines
11 KiB
Java
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package com.guo.learningprogresstracker.utils;
import lombok.extern.slf4j.Slf4j;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URI;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.security.cert.X509Certificate;
import java.util.*;
import java.util.concurrent.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;
import java.util.zip.InflaterInputStream;
@Slf4j
public class TitleFetcher {
private static final Pattern TITLE_PATTERN = Pattern.compile(
"<title[^>]*>([^<]+)</title>", Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
private static final Pattern CHARSET_PATTERN = Pattern.compile(
"charset=([\\w\\-]+)", Pattern.CASE_INSENSITIVE);
private static final int MAX_REDIRECTS = 5;
private static final int TIMEOUT_MS = 8_000;
private static volatile boolean sslRelaxed;
static {
try {
TrustManager[] trustAll = new TrustManager[]{
new X509TrustManager() {
public X509Certificate[] getAcceptedIssuers() { return new X509Certificate[0]; }
public void checkClientTrusted(X509Certificate[] c, String a) {}
public void checkServerTrusted(X509Certificate[] c, String a) {}
}
};
SSLContext sc = SSLContext.getInstance("TLS");
sc.init(null, trustAll, new java.security.SecureRandom());
HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
HttpsURLConnection.setDefaultHostnameVerifier((hostname, session) -> true);
sslRelaxed = true;
} catch (Exception e) {
log.warn("无法配置宽松 SSL: {}", e.getMessage());
}
}
public static String fetchTitle(String url) {
if (url == null || url.isBlank()) return null;
String lower = url.toLowerCase();
if (!lower.startsWith("http://") && !lower.startsWith("https://")) return null;
String currentUrl = url;
for (int hop = 0; hop <= MAX_REDIRECTS; hop++) {
String title = doFetch(currentUrl);
if (title != null) return title;
// 检查是否需要跟随重定向
String redirect = getRedirect(currentUrl);
if (redirect != null && !redirect.equals(currentUrl)) {
currentUrl = redirect;
continue;
}
return null;
}
return null;
}
private static String doFetch(String url) {
HttpURLConnection conn = null;
try {
conn = (HttpURLConnection) URI.create(url).toURL().openConnection();
if (conn instanceof HttpsURLConnection) {
// SSL 已全局宽松配置
}
conn.setConnectTimeout(TIMEOUT_MS);
conn.setReadTimeout(TIMEOUT_MS);
conn.setRequestMethod("GET");
conn.setInstanceFollowRedirects(false); // 手动处理重定向
conn.setRequestProperty("Connection", "close");
conn.setRequestProperty("User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36");
conn.setRequestProperty("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
conn.setRequestProperty("Accept-Encoding", "gzip, deflate");
conn.setRequestProperty("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8");
int status = conn.getResponseCode();
if (status >= 200 && status < 400) {
String contentType = conn.getContentType();
if (contentType != null && !contentType.toLowerCase().contains("text/html")
&& !contentType.toLowerCase().contains("application/xhtml")) {
if (!contentType.toLowerCase().contains("text/")) return null;
}
Charset charset = detectCharset(contentType);
String body = readBody(conn, charset);
if (body == null) return null;
Matcher matcher = TITLE_PATTERN.matcher(body);
if (matcher.find()) {
String title = matcher.group(1).trim();
title = title.replace("&amp;", "&")
.replace("&lt;", "<")
.replace("&gt;", ">")
.replace("&quot;", "\"")
.replace("&#39;", "'")
.replace("&nbsp;", " ")
.replaceAll("\\s+", " ").trim();
return title.isEmpty() ? null : title;
}
}
return null;
} catch (Exception e) {
log.debug("获取页面标题失败: url={}, error={}", url, e.getMessage());
return null;
} finally {
if (conn != null) conn.disconnect();
}
}
/** 获取重定向地址,支持跨协议跳转 */
private static String getRedirect(String url) {
HttpURLConnection conn = null;
try {
conn = (HttpURLConnection) URI.create(url).toURL().openConnection();
conn.setConnectTimeout(TIMEOUT_MS);
conn.setReadTimeout(TIMEOUT_MS);
conn.setRequestMethod("HEAD");
conn.setInstanceFollowRedirects(false);
conn.setRequestProperty("User-Agent",
"Mozilla/5.0 (compatible; LPT/1.0)");
int status = conn.getResponseCode();
if (status == HttpURLConnection.HTTP_MOVED_PERM // 301
|| status == HttpURLConnection.HTTP_MOVED_TEMP // 302
|| status == HttpURLConnection.HTTP_SEE_OTHER // 303
|| status == 307
|| status == 308) {
String location = conn.getHeaderField("Location");
if (location != null && !location.isBlank()) {
// 处理相对路径
if (!location.toLowerCase().startsWith("http")) {
URI base = URI.create(url);
location = base.resolve(location).toString();
}
// 跨协议切换(HTTP→HTTPS)也允许
if (location.toLowerCase().startsWith("http://")
|| location.toLowerCase().startsWith("https://")) {
return location;
}
}
}
return null;
} catch (Exception e) {
return null;
} finally {
if (conn != null) conn.disconnect();
}
}
/**
* 将文本中的裸 URL 替换为 [标题](url) 格式(已有 [text](url) 的保持不变)。
* 并行抓取标题,整体超时 15 秒。
*/
public static String embedTitles(String text) {
if (text == null || text.isBlank()) return text;
// 1) 保护已有 [text](url)
List<String> protectedLinks = new ArrayList<>();
Matcher mdLink = Pattern.compile("\\[([^\\]]+)\\]\\(([^)]+)\\)").matcher(text);
StringBuffer sb = new StringBuffer();
while (mdLink.find()) {
protectedLinks.add(mdLink.group(0));
mdLink.appendReplacement(sb, "__MDLINK_" + (protectedLinks.size() - 1) + "__");
}
mdLink.appendTail(sb);
String work = sb.toString();
// 2) 提取裸 URL 并去重
Set<String> bareUrls = new LinkedHashSet<>();
Matcher um = Pattern.compile("https?://[^\\s)\u3001\uFF09\u300D\u300B<>]+").matcher(work);
while (um.find()) {
String url = um.group().replaceAll("[.。,,;;!!??)】」』\\]]+$", "");
bareUrls.add(url);
}
if (bareUrls.isEmpty()) return text;
// 3) 并行抓取标题
Map<String, String> titleMap = new ConcurrentHashMap<>();
@SuppressWarnings("unchecked")
CompletableFuture<Void>[] futures = new CompletableFuture[bareUrls.size()];
int fi = 0;
for (String url : bareUrls) {
final String u = url;
futures[fi++] = CompletableFuture.runAsync(() -> {
String title = fetchTitle(u);
if (title != null) titleMap.put(u, title);
});
}
try {
CompletableFuture.allOf(futures).get(15, TimeUnit.SECONDS);
} catch (Exception e) {
log.debug("批量获取标题超时或失败: {}", e.getMessage());
}
// 4) 替换裸 URL → [标题](url)
for (String url : bareUrls) {
String title = titleMap.get(url);
if (title != null) {
work = work.replace(url, "[" + title.replace("]", "\\]") + "](" + url + ")");
}
}
// 5) 还原 [text](url)
for (int i = 0; i < protectedLinks.size(); i++) {
work = work.replace("__MDLINK_" + i + "__", protectedLinks.get(i));
}
return work;
}
private static Charset detectCharset(String contentType) {
if (contentType != null) {
Matcher m = CHARSET_PATTERN.matcher(contentType);
if (m.find()) {
try { return Charset.forName(m.group(1)); } catch (Exception ignored) {}
}
}
return StandardCharsets.UTF_8;
}
private static String readBody(HttpURLConnection conn, Charset charset) throws IOException {
InputStream is;
try {
is = conn.getInputStream();
} catch (IOException e) {
is = conn.getErrorStream();
}
if (is == null) return null;
String encoding = conn.getContentEncoding();
try {
if ("gzip".equalsIgnoreCase(encoding)) {
is = new GZIPInputStream(is);
} else if ("deflate".equalsIgnoreCase(encoding)) {
is = new InflaterInputStream(is);
}
} catch (IOException ignored) {
try { is.close(); } catch (Exception ignored2) {}
is = conn.getInputStream();
}
byte[] buf = new byte[65536];
int total = 0;
try {
int n;
while (total < buf.length && (n = is.read(buf, total, buf.length - total)) != -1) {
total += n;
}
} finally {
try { is.close(); } catch (Exception ignored) {}
}
return new String(buf, 0, total, charset);
}
}