package com.guo.learningprogresstracker.utils; import lombok.extern.slf4j.Slf4j; import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; import java.io.IOException; import java.io.InputStream; import java.net.HttpURLConnection; import java.net.URI; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.security.cert.X509Certificate; import java.util.*; import java.util.concurrent.*; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; import java.util.zip.InflaterInputStream; @Slf4j public class TitleFetcher { private static final Pattern TITLE_PATTERN = Pattern.compile( "]*>([^<]+)", Pattern.CASE_INSENSITIVE | Pattern.DOTALL); private static final Pattern CHARSET_PATTERN = Pattern.compile( "charset=([\\w\\-]+)", Pattern.CASE_INSENSITIVE); private static final int MAX_REDIRECTS = 5; private static final int TIMEOUT_MS = 8_000; private static volatile boolean sslRelaxed; static { try { TrustManager[] trustAll = new TrustManager[]{ new X509TrustManager() { public X509Certificate[] getAcceptedIssuers() { return new X509Certificate[0]; } public void checkClientTrusted(X509Certificate[] c, String a) {} public void checkServerTrusted(X509Certificate[] c, String a) {} } }; SSLContext sc = SSLContext.getInstance("TLS"); sc.init(null, trustAll, new java.security.SecureRandom()); HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); HttpsURLConnection.setDefaultHostnameVerifier((hostname, session) -> true); sslRelaxed = true; } catch (Exception e) { log.warn("无法配置宽松 SSL: {}", e.getMessage()); } } public static String fetchTitle(String url) { if (url == null || url.isBlank()) return null; String lower = url.toLowerCase(); if (!lower.startsWith("http://") && !lower.startsWith("https://")) return null; String currentUrl = url; for (int hop = 0; hop <= MAX_REDIRECTS; hop++) { String title = doFetch(currentUrl); if (title != null) return title; // 检查是否需要跟随重定向 String redirect = getRedirect(currentUrl); if (redirect != null && !redirect.equals(currentUrl)) { currentUrl = redirect; continue; } return null; } return null; } private static String doFetch(String url) { HttpURLConnection conn = null; try { conn = (HttpURLConnection) URI.create(url).toURL().openConnection(); if (conn instanceof HttpsURLConnection) { // SSL 已全局宽松配置 } conn.setConnectTimeout(TIMEOUT_MS); conn.setReadTimeout(TIMEOUT_MS); conn.setRequestMethod("GET"); conn.setInstanceFollowRedirects(false); // 手动处理重定向 conn.setRequestProperty("Connection", "close"); conn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"); conn.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); conn.setRequestProperty("Accept-Encoding", "gzip, deflate"); conn.setRequestProperty("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"); int status = conn.getResponseCode(); if (status >= 200 && status < 400) { String contentType = conn.getContentType(); if (contentType != null && !contentType.toLowerCase().contains("text/html") && !contentType.toLowerCase().contains("application/xhtml")) { if (!contentType.toLowerCase().contains("text/")) return null; } Charset charset = detectCharset(contentType); String body = readBody(conn, charset); if (body == null) return null; Matcher matcher = TITLE_PATTERN.matcher(body); if (matcher.find()) { String title = matcher.group(1).trim(); title = title.replace("&", "&") .replace("<", "<") .replace(">", ">") .replace(""", "\"") .replace("'", "'") .replace(" ", " ") .replaceAll("\\s+", " ").trim(); return title.isEmpty() ? null : title; } } return null; } catch (Exception e) { log.debug("获取页面标题失败: url={}, error={}", url, e.getMessage()); return null; } finally { if (conn != null) conn.disconnect(); } } /** 获取重定向地址,支持跨协议跳转 */ private static String getRedirect(String url) { HttpURLConnection conn = null; try { conn = (HttpURLConnection) URI.create(url).toURL().openConnection(); conn.setConnectTimeout(TIMEOUT_MS); conn.setReadTimeout(TIMEOUT_MS); conn.setRequestMethod("HEAD"); conn.setInstanceFollowRedirects(false); conn.setRequestProperty("User-Agent", "Mozilla/5.0 (compatible; LPT/1.0)"); int status = conn.getResponseCode(); if (status == HttpURLConnection.HTTP_MOVED_PERM // 301 || status == HttpURLConnection.HTTP_MOVED_TEMP // 302 || status == HttpURLConnection.HTTP_SEE_OTHER // 303 || status == 307 || status == 308) { String location = conn.getHeaderField("Location"); if (location != null && !location.isBlank()) { // 处理相对路径 if (!location.toLowerCase().startsWith("http")) { URI base = URI.create(url); location = base.resolve(location).toString(); } // 跨协议切换(HTTP→HTTPS)也允许 if (location.toLowerCase().startsWith("http://") || location.toLowerCase().startsWith("https://")) { return location; } } } return null; } catch (Exception e) { return null; } finally { if (conn != null) conn.disconnect(); } } /** * 将文本中的裸 URL 替换为 [标题](url) 格式(已有 [text](url) 的保持不变)。 * 并行抓取标题,整体超时 15 秒。 */ public static String embedTitles(String text) { if (text == null || text.isBlank()) return text; // 1) 保护已有 [text](url) List protectedLinks = new ArrayList<>(); Matcher mdLink = Pattern.compile("\\[([^\\]]+)\\]\\(([^)]+)\\)").matcher(text); StringBuffer sb = new StringBuffer(); while (mdLink.find()) { protectedLinks.add(mdLink.group(0)); mdLink.appendReplacement(sb, "__MDLINK_" + (protectedLinks.size() - 1) + "__"); } mdLink.appendTail(sb); String work = sb.toString(); // 2) 提取裸 URL 并去重 Set bareUrls = new LinkedHashSet<>(); Matcher um = Pattern.compile("https?://[^\\s)\u3001\uFF09\u300D\u300B<>]+").matcher(work); while (um.find()) { String url = um.group().replaceAll("[.。,,;;!!??)】」』\\]]+$", ""); bareUrls.add(url); } if (bareUrls.isEmpty()) return text; // 3) 并行抓取标题 Map titleMap = new ConcurrentHashMap<>(); @SuppressWarnings("unchecked") CompletableFuture[] futures = new CompletableFuture[bareUrls.size()]; int fi = 0; for (String url : bareUrls) { final String u = url; futures[fi++] = CompletableFuture.runAsync(() -> { String title = fetchTitle(u); if (title != null) titleMap.put(u, title); }); } try { CompletableFuture.allOf(futures).get(15, TimeUnit.SECONDS); } catch (Exception e) { log.debug("批量获取标题超时或失败: {}", e.getMessage()); } // 4) 替换裸 URL → [标题](url) for (String url : bareUrls) { String title = titleMap.get(url); if (title != null) { work = work.replace(url, "[" + title.replace("]", "\\]") + "](" + url + ")"); } } // 5) 还原 [text](url) for (int i = 0; i < protectedLinks.size(); i++) { work = work.replace("__MDLINK_" + i + "__", protectedLinks.get(i)); } return work; } private static Charset detectCharset(String contentType) { if (contentType != null) { Matcher m = CHARSET_PATTERN.matcher(contentType); if (m.find()) { try { return Charset.forName(m.group(1)); } catch (Exception ignored) {} } } return StandardCharsets.UTF_8; } private static String readBody(HttpURLConnection conn, Charset charset) throws IOException { InputStream is; try { is = conn.getInputStream(); } catch (IOException e) { is = conn.getErrorStream(); } if (is == null) return null; String encoding = conn.getContentEncoding(); try { if ("gzip".equalsIgnoreCase(encoding)) { is = new GZIPInputStream(is); } else if ("deflate".equalsIgnoreCase(encoding)) { is = new InflaterInputStream(is); } } catch (IOException ignored) { try { is.close(); } catch (Exception ignored2) {} is = conn.getInputStream(); } byte[] buf = new byte[65536]; int total = 0; try { int n; while (total < buf.length && (n = is.read(buf, total, buf.length - total)) != -1) { total += n; } } finally { try { is.close(); } catch (Exception ignored) {} } return new String(buf, 0, total, charset); } }