feat: 新增 URL 标题抓取工具

- UtilsController 提供 /utils/fetch-title 接口，供前端获取网页标题 - TitleFetcher 工具类支持 HTTP/HTTPS 请求、重定向跟踪、gzip/deflate 解压 - 支持 SSL 宽松配置、编码自动检测、批量并行抓取 - 支持裸 URL 自动嵌入为 [标题](url) Markdown 格式
2026-07-12 13:43:58 +08:00
parent ac0154c1fb
commit 53a6781092
2 changed files with 299 additions and 0 deletions
@@ -0,0 +1,26 @@
+package com.guo.learningprogresstracker.controller;
+
+import com.guo.learningprogresstracker.entity.CommonResult;
+import com.guo.learningprogresstracker.utils.TitleFetcher;
+import io.swagger.v3.oas.annotations.Operation;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.Map;
+
+@RestController
+@RequestMapping("/utils")
+public class UtilsController {
+
+    @GetMapping("/fetch-title")
+    @Operation(summary = "获取指定URL的页面标题")
+    public CommonResult<Map<String, String>> fetchTitle(@RequestParam String url) {
+        if (url == null || url.isBlank()) {
+            return CommonResult.error("url 参数不能为空");
+        }
+        String title = TitleFetcher.fetchTitle(url);
+        return CommonResult.success(Map.of("title", title != null ? title : url));
+    }
+}
@@ -0,0 +1,273 @@
+package com.guo.learningprogresstracker.utils;
+
+import lombok.extern.slf4j.Slf4j;
+
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.security.cert.X509Certificate;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.InflaterInputStream;
+
+@Slf4j
+public class TitleFetcher {
+
+    private static final Pattern TITLE_PATTERN = Pattern.compile(
+            "<title[^>]*>([^<]+)</title>", Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
+    private static final Pattern CHARSET_PATTERN = Pattern.compile(
+            "charset=([\\w\\-]+)", Pattern.CASE_INSENSITIVE);
+    private static final int MAX_REDIRECTS = 5;
+    private static final int TIMEOUT_MS = 8_000;
+
+    private static volatile boolean sslRelaxed;
+
+    static {
+        try {
+            TrustManager[] trustAll = new TrustManager[]{
+                    new X509TrustManager() {
+                        public X509Certificate[] getAcceptedIssuers() { return new X509Certificate[0]; }
+                        public void checkClientTrusted(X509Certificate[] c, String a) {}
+                        public void checkServerTrusted(X509Certificate[] c, String a) {}
+                    }
+            };
+            SSLContext sc = SSLContext.getInstance("TLS");
+            sc.init(null, trustAll, new java.security.SecureRandom());
+            HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
+            HttpsURLConnection.setDefaultHostnameVerifier((hostname, session) -> true);
+            sslRelaxed = true;
+        } catch (Exception e) {
+            log.warn("无法配置宽松 SSL: {}", e.getMessage());
+        }
+    }
+
+    public static String fetchTitle(String url) {
+        if (url == null || url.isBlank()) return null;
+        String lower = url.toLowerCase();
+        if (!lower.startsWith("http://") && !lower.startsWith("https://")) return null;
+
+        String currentUrl = url;
+        for (int hop = 0; hop <= MAX_REDIRECTS; hop++) {
+            String title = doFetch(currentUrl);
+            if (title != null) return title;
+
+            // 检查是否需要跟随重定向
+            String redirect = getRedirect(currentUrl);
+            if (redirect != null && !redirect.equals(currentUrl)) {
+                currentUrl = redirect;
+                continue;
+            }
+            return null;
+        }
+        return null;
+    }
+
+    private static String doFetch(String url) {
+        HttpURLConnection conn = null;
+        try {
+            conn = (HttpURLConnection) URI.create(url).toURL().openConnection();
+            if (conn instanceof HttpsURLConnection) {
+                // SSL 已全局宽松配置
+            }
+            conn.setConnectTimeout(TIMEOUT_MS);
+            conn.setReadTimeout(TIMEOUT_MS);
+            conn.setRequestMethod("GET");
+            conn.setInstanceFollowRedirects(false); // 手动处理重定向
+            conn.setRequestProperty("Connection", "close");
+            conn.setRequestProperty("User-Agent",
+                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
+                    "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36");
+            conn.setRequestProperty("Accept",
+                    "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
+            conn.setRequestProperty("Accept-Encoding", "gzip, deflate");
+            conn.setRequestProperty("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8");
+
+            int status = conn.getResponseCode();
+            if (status >= 200 && status < 400) {
+                String contentType = conn.getContentType();
+                if (contentType != null && !contentType.toLowerCase().contains("text/html")
+                        && !contentType.toLowerCase().contains("application/xhtml")) {
+                    if (!contentType.toLowerCase().contains("text/")) return null;
+                }
+
+                Charset charset = detectCharset(contentType);
+                String body = readBody(conn, charset);
+                if (body == null) return null;
+
+                Matcher matcher = TITLE_PATTERN.matcher(body);
+                if (matcher.find()) {
+                    String title = matcher.group(1).trim();
+                    title = title.replace("&amp;", "&")
+                            .replace("&lt;", "<")
+                            .replace("&gt;", ">")
+                            .replace("&quot;", "\"")
+                            .replace("&#39;", "'")
+                            .replace("&nbsp;", " ")
+                            .replaceAll("\\s+", " ").trim();
+                    return title.isEmpty() ? null : title;
+                }
+            }
+            return null;
+        } catch (Exception e) {
+            log.debug("获取页面标题失败: url={}, error={}", url, e.getMessage());
+            return null;
+        } finally {
+            if (conn != null) conn.disconnect();
+        }
+    }
+
+    /** 获取重定向地址，支持跨协议跳转 */
+    private static String getRedirect(String url) {
+        HttpURLConnection conn = null;
+        try {
+            conn = (HttpURLConnection) URI.create(url).toURL().openConnection();
+            conn.setConnectTimeout(TIMEOUT_MS);
+            conn.setReadTimeout(TIMEOUT_MS);
+            conn.setRequestMethod("HEAD");
+            conn.setInstanceFollowRedirects(false);
+            conn.setRequestProperty("User-Agent",
+                    "Mozilla/5.0 (compatible; LPT/1.0)");
+
+            int status = conn.getResponseCode();
+            if (status == HttpURLConnection.HTTP_MOVED_PERM   // 301
+                    || status == HttpURLConnection.HTTP_MOVED_TEMP  // 302
+                    || status == HttpURLConnection.HTTP_SEE_OTHER   // 303
+                    || status == 307
+                    || status == 308) {
+                String location = conn.getHeaderField("Location");
+                if (location != null && !location.isBlank()) {
+                    // 处理相对路径
+                    if (!location.toLowerCase().startsWith("http")) {
+                        URI base = URI.create(url);
+                        location = base.resolve(location).toString();
+                    }
+                    // 跨协议切换（HTTP→HTTPS）也允许
+                    if (location.toLowerCase().startsWith("http://")
+                            || location.toLowerCase().startsWith("https://")) {
+                        return location;
+                    }
+                }
+            }
+            return null;
+        } catch (Exception e) {
+            return null;
+        } finally {
+            if (conn != null) conn.disconnect();
+        }
+    }
+
+    /**
+     * 将文本中的裸 URL 替换为 [标题](url) 格式（已有 [text](url) 的保持不变）。
+     * 并行抓取标题，整体超时 15 秒。
+     */
+    public static String embedTitles(String text) {
+        if (text == null || text.isBlank()) return text;
+
+        // 1) 保护已有 [text](url)
+        List<String> protectedLinks = new ArrayList<>();
+        Matcher mdLink = Pattern.compile("\\[([^\\]]+)\\]\\(([^)]+)\\)").matcher(text);
+        StringBuffer sb = new StringBuffer();
+        while (mdLink.find()) {
+            protectedLinks.add(mdLink.group(0));
+            mdLink.appendReplacement(sb, "__MDLINK_" + (protectedLinks.size() - 1) + "__");
+        }
+        mdLink.appendTail(sb);
+        String work = sb.toString();
+
+        // 2) 提取裸 URL 并去重
+        Set<String> bareUrls = new LinkedHashSet<>();
+        Matcher um = Pattern.compile("https?://[^\\s)\u3001\uFF09\u300D\u300B<>]+").matcher(work);
+        while (um.find()) {
+            String url = um.group().replaceAll("[.。,，;；!！?？）】」』\\]]+$", "");
+            bareUrls.add(url);
+        }
+        if (bareUrls.isEmpty()) return text;
+
+        // 3) 并行抓取标题
+        Map<String, String> titleMap = new ConcurrentHashMap<>();
+        @SuppressWarnings("unchecked")
+        CompletableFuture<Void>[] futures = new CompletableFuture[bareUrls.size()];
+        int fi = 0;
+        for (String url : bareUrls) {
+            final String u = url;
+            futures[fi++] = CompletableFuture.runAsync(() -> {
+                String title = fetchTitle(u);
+                if (title != null) titleMap.put(u, title);
+            });
+        }
+        try {
+            CompletableFuture.allOf(futures).get(15, TimeUnit.SECONDS);
+        } catch (Exception e) {
+            log.debug("批量获取标题超时或失败: {}", e.getMessage());
+        }
+
+        // 4) 替换裸 URL → [标题](url)
+        for (String url : bareUrls) {
+            String title = titleMap.get(url);
+            if (title != null) {
+                work = work.replace(url, "[" + title.replace("]", "\\]") + "](" + url + ")");
+            }
+        }
+
+        // 5) 还原 [text](url)
+        for (int i = 0; i < protectedLinks.size(); i++) {
+            work = work.replace("__MDLINK_" + i + "__", protectedLinks.get(i));
+        }
+
+        return work;
+    }
+
+    private static Charset detectCharset(String contentType) {
+        if (contentType != null) {
+            Matcher m = CHARSET_PATTERN.matcher(contentType);
+            if (m.find()) {
+                try { return Charset.forName(m.group(1)); } catch (Exception ignored) {}
+            }
+        }
+        return StandardCharsets.UTF_8;
+    }
+
+    private static String readBody(HttpURLConnection conn, Charset charset) throws IOException {
+        InputStream is;
+        try {
+            is = conn.getInputStream();
+        } catch (IOException e) {
+            is = conn.getErrorStream();
+        }
+        if (is == null) return null;
+
+        String encoding = conn.getContentEncoding();
+        try {
+            if ("gzip".equalsIgnoreCase(encoding)) {
+                is = new GZIPInputStream(is);
+            } else if ("deflate".equalsIgnoreCase(encoding)) {
+                is = new InflaterInputStream(is);
+            }
+        } catch (IOException ignored) {
+            try { is.close(); } catch (Exception ignored2) {}
+            is = conn.getInputStream();
+        }
+
+        byte[] buf = new byte[65536];
+        int total = 0;
+        try {
+            int n;
+            while (total < buf.length && (n = is.read(buf, total, buf.length - total)) != -1) {
+                total += n;
+            }
+        } finally {
+            try { is.close(); } catch (Exception ignored) {}
+        }
+        return new String(buf, 0, total, charset);
+    }
+}