diff --git a/src/main/java/com/guo/learningprogresstracker/controller/UtilsController.java b/src/main/java/com/guo/learningprogresstracker/controller/UtilsController.java new file mode 100644 index 0000000..df8c44b --- /dev/null +++ b/src/main/java/com/guo/learningprogresstracker/controller/UtilsController.java @@ -0,0 +1,26 @@ +package com.guo.learningprogresstracker.controller; + +import com.guo.learningprogresstracker.entity.CommonResult; +import com.guo.learningprogresstracker.utils.TitleFetcher; +import io.swagger.v3.oas.annotations.Operation; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Map; + +@RestController +@RequestMapping("/utils") +public class UtilsController { + + @GetMapping("/fetch-title") + @Operation(summary = "获取指定URL的页面标题") + public CommonResult> fetchTitle(@RequestParam String url) { + if (url == null || url.isBlank()) { + return CommonResult.error("url 参数不能为空"); + } + String title = TitleFetcher.fetchTitle(url); + return CommonResult.success(Map.of("title", title != null ? title : url)); + } +} diff --git a/src/main/java/com/guo/learningprogresstracker/utils/TitleFetcher.java b/src/main/java/com/guo/learningprogresstracker/utils/TitleFetcher.java new file mode 100644 index 0000000..4700ad5 --- /dev/null +++ b/src/main/java/com/guo/learningprogresstracker/utils/TitleFetcher.java @@ -0,0 +1,273 @@ +package com.guo.learningprogresstracker.utils; + +import lombok.extern.slf4j.Slf4j; + +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URI; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.security.cert.X509Certificate; +import java.util.*; +import java.util.concurrent.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.zip.GZIPInputStream; +import java.util.zip.InflaterInputStream; + +@Slf4j +public class TitleFetcher { + + private static final Pattern TITLE_PATTERN = Pattern.compile( + "]*>([^<]+)", Pattern.CASE_INSENSITIVE | Pattern.DOTALL); + private static final Pattern CHARSET_PATTERN = Pattern.compile( + "charset=([\\w\\-]+)", Pattern.CASE_INSENSITIVE); + private static final int MAX_REDIRECTS = 5; + private static final int TIMEOUT_MS = 8_000; + + private static volatile boolean sslRelaxed; + + static { + try { + TrustManager[] trustAll = new TrustManager[]{ + new X509TrustManager() { + public X509Certificate[] getAcceptedIssuers() { return new X509Certificate[0]; } + public void checkClientTrusted(X509Certificate[] c, String a) {} + public void checkServerTrusted(X509Certificate[] c, String a) {} + } + }; + SSLContext sc = SSLContext.getInstance("TLS"); + sc.init(null, trustAll, new java.security.SecureRandom()); + HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); + HttpsURLConnection.setDefaultHostnameVerifier((hostname, session) -> true); + sslRelaxed = true; + } catch (Exception e) { + log.warn("无法配置宽松 SSL: {}", e.getMessage()); + } + } + + public static String fetchTitle(String url) { + if (url == null || url.isBlank()) return null; + String lower = url.toLowerCase(); + if (!lower.startsWith("http://") && !lower.startsWith("https://")) return null; + + String currentUrl = url; + for (int hop = 0; hop <= MAX_REDIRECTS; hop++) { + String title = doFetch(currentUrl); + if (title != null) return title; + + // 检查是否需要跟随重定向 + String redirect = getRedirect(currentUrl); + if (redirect != null && !redirect.equals(currentUrl)) { + currentUrl = redirect; + continue; + } + return null; + } + return null; + } + + private static String doFetch(String url) { + HttpURLConnection conn = null; + try { + conn = (HttpURLConnection) URI.create(url).toURL().openConnection(); + if (conn instanceof HttpsURLConnection) { + // SSL 已全局宽松配置 + } + conn.setConnectTimeout(TIMEOUT_MS); + conn.setReadTimeout(TIMEOUT_MS); + conn.setRequestMethod("GET"); + conn.setInstanceFollowRedirects(false); // 手动处理重定向 + conn.setRequestProperty("Connection", "close"); + conn.setRequestProperty("User-Agent", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + + "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"); + conn.setRequestProperty("Accept", + "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); + conn.setRequestProperty("Accept-Encoding", "gzip, deflate"); + conn.setRequestProperty("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"); + + int status = conn.getResponseCode(); + if (status >= 200 && status < 400) { + String contentType = conn.getContentType(); + if (contentType != null && !contentType.toLowerCase().contains("text/html") + && !contentType.toLowerCase().contains("application/xhtml")) { + if (!contentType.toLowerCase().contains("text/")) return null; + } + + Charset charset = detectCharset(contentType); + String body = readBody(conn, charset); + if (body == null) return null; + + Matcher matcher = TITLE_PATTERN.matcher(body); + if (matcher.find()) { + String title = matcher.group(1).trim(); + title = title.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace(""", "\"") + .replace("'", "'") + .replace(" ", " ") + .replaceAll("\\s+", " ").trim(); + return title.isEmpty() ? null : title; + } + } + return null; + } catch (Exception e) { + log.debug("获取页面标题失败: url={}, error={}", url, e.getMessage()); + return null; + } finally { + if (conn != null) conn.disconnect(); + } + } + + /** 获取重定向地址，支持跨协议跳转 */ + private static String getRedirect(String url) { + HttpURLConnection conn = null; + try { + conn = (HttpURLConnection) URI.create(url).toURL().openConnection(); + conn.setConnectTimeout(TIMEOUT_MS); + conn.setReadTimeout(TIMEOUT_MS); + conn.setRequestMethod("HEAD"); + conn.setInstanceFollowRedirects(false); + conn.setRequestProperty("User-Agent", + "Mozilla/5.0 (compatible; LPT/1.0)"); + + int status = conn.getResponseCode(); + if (status == HttpURLConnection.HTTP_MOVED_PERM // 301 + || status == HttpURLConnection.HTTP_MOVED_TEMP // 302 + || status == HttpURLConnection.HTTP_SEE_OTHER // 303 + || status == 307 + || status == 308) { + String location = conn.getHeaderField("Location"); + if (location != null && !location.isBlank()) { + // 处理相对路径 + if (!location.toLowerCase().startsWith("http")) { + URI base = URI.create(url); + location = base.resolve(location).toString(); + } + // 跨协议切换（HTTP→HTTPS）也允许 + if (location.toLowerCase().startsWith("http://") + || location.toLowerCase().startsWith("https://")) { + return location; + } + } + } + return null; + } catch (Exception e) { + return null; + } finally { + if (conn != null) conn.disconnect(); + } + } + + /** + * 将文本中的裸 URL 替换为 [标题](url) 格式（已有 [text](url) 的保持不变）。 + * 并行抓取标题，整体超时 15 秒。 + */ + public static String embedTitles(String text) { + if (text == null || text.isBlank()) return text; + + // 1) 保护已有 [text](url) + List protectedLinks = new ArrayList<>(); + Matcher mdLink = Pattern.compile("\\[([^\\]]+)\\]\\(([^)]+)\\)").matcher(text); + StringBuffer sb = new StringBuffer(); + while (mdLink.find()) { + protectedLinks.add(mdLink.group(0)); + mdLink.appendReplacement(sb, "__MDLINK_" + (protectedLinks.size() - 1) + "__"); + } + mdLink.appendTail(sb); + String work = sb.toString(); + + // 2) 提取裸 URL 并去重 + Set bareUrls = new LinkedHashSet<>(); + Matcher um = Pattern.compile("https?://[^\\s)\u3001\uFF09\u300D\u300B<>]+").matcher(work); + while (um.find()) { + String url = um.group().replaceAll("[.。,，;；!！?？）】」』\\]]+$", ""); + bareUrls.add(url); + } + if (bareUrls.isEmpty()) return text; + + // 3) 并行抓取标题 + Map titleMap = new ConcurrentHashMap<>(); + @SuppressWarnings("unchecked") + CompletableFuture[] futures = new CompletableFuture[bareUrls.size()]; + int fi = 0; + for (String url : bareUrls) { + final String u = url; + futures[fi++] = CompletableFuture.runAsync(() -> { + String title = fetchTitle(u); + if (title != null) titleMap.put(u, title); + }); + } + try { + CompletableFuture.allOf(futures).get(15, TimeUnit.SECONDS); + } catch (Exception e) { + log.debug("批量获取标题超时或失败: {}", e.getMessage()); + } + + // 4) 替换裸 URL → [标题](url) + for (String url : bareUrls) { + String title = titleMap.get(url); + if (title != null) { + work = work.replace(url, "[" + title.replace("]", "\\]") + "](" + url + ")"); + } + } + + // 5) 还原 [text](url) + for (int i = 0; i < protectedLinks.size(); i++) { + work = work.replace("__MDLINK_" + i + "__", protectedLinks.get(i)); + } + + return work; + } + + private static Charset detectCharset(String contentType) { + if (contentType != null) { + Matcher m = CHARSET_PATTERN.matcher(contentType); + if (m.find()) { + try { return Charset.forName(m.group(1)); } catch (Exception ignored) {} + } + } + return StandardCharsets.UTF_8; + } + + private static String readBody(HttpURLConnection conn, Charset charset) throws IOException { + InputStream is; + try { + is = conn.getInputStream(); + } catch (IOException e) { + is = conn.getErrorStream(); + } + if (is == null) return null; + + String encoding = conn.getContentEncoding(); + try { + if ("gzip".equalsIgnoreCase(encoding)) { + is = new GZIPInputStream(is); + } else if ("deflate".equalsIgnoreCase(encoding)) { + is = new InflaterInputStream(is); + } + } catch (IOException ignored) { + try { is.close(); } catch (Exception ignored2) {} + is = conn.getInputStream(); + } + + byte[] buf = new byte[65536]; + int total = 0; + try { + int n; + while (total < buf.length && (n = is.read(buf, total, buf.length - total)) != -1) { + total += n; + } + } finally { + try { is.close(); } catch (Exception ignored) {} + } + return new String(buf, 0, total, charset); + } +}