/** * SSRF 防护:校验「服务端主动抓取」的目标地址是否安全。 * * 背景:`/fetch-title` 由服务端抓取调用方给的 URL。如果不做限制,调用方可以拿它探测 * 内网服务与云元数据地址(169.254.169.254),把本服务当成内网跳板。 * * 防护分四层,缺一层都能绕过: * 1. 协议只允许 http/https; * 2. 端口只允许 http/https 默认端口(外加显式配置的额外端口),避免被用来探测 Redis/MySQL 等内网服务; * 3. 解析域名后校验**每一个**解析结果,而不是只校验字面量(防 DNS 解析到内网、防 DNS rebinding); * 4. 重定向逐跳校验,避免「首跳合法、次跳内网」。 */ import { lookup } from "node:dns/promises"; /** 允许的协议 */ const ALLOWED_PROTOCOLS = new Set(["http:", "https:"]); /** 默认允许的端口 */ const DEFAULT_ALLOWED_PORTS = new Set(["80", "443"]); /** 响应体读取上限(字节):标题通常在头部,不必读完整个页面 */ export const MAX_RESPONSE_BYTES = 512 * 1024; /** 最大重定向跳数 */ export const MAX_REDIRECTS = 3; export class UnsafeUrlError extends Error { constructor(message: string) { super(message); this.name = "UnsafeUrlError"; } } /** 解析额外允许的端口:环境变量 LPT_AI_FETCH_ALLOWED_PORTS,逗号分隔 */ function allowedPorts(): Set { const extra = (process.env.LPT_AI_FETCH_ALLOWED_PORTS ?? "") .split(",") .map((p) => p.trim()) .filter((p) => /^\d+$/.test(p)); return new Set([...DEFAULT_ALLOWED_PORTS, ...extra]); } /** * 判断一个 IP 字面量是否属于「不可主动访问」的地址段。 * * 覆盖 IPv4 私网/回环/链路本地(含云元数据 169.254.169.254)/CGNAT/组播/保留段, * 以及 IPv6 回环、未指定、ULA、链路本地、组播,并识别 IPv4-mapped 与 NAT64 里内嵌的 IPv4。 */ export function isBlockedAddress(address: string): boolean { const ip = address.trim().toLowerCase(); if (!ip) return true; // IPv6:去掉可能的 zone id(fe80::1%eth0) const bare = ip.includes("%") ? ip.slice(0, ip.indexOf("%")) : ip; if (bare.includes(":")) { return isBlockedIpv6(bare); } return isBlockedIpv4(bare); } function isBlockedIpv4(ip: string): boolean { const parts = ip.split("."); if (parts.length !== 4) return true; // 不是合法 IPv4,按不安全处理 const octets = parts.map((p) => Number(p)); if (octets.some((n) => !Number.isInteger(n) || n < 0 || n > 255)) return true; const [a, b] = octets; if (a === 0) return true; // 0.0.0.0/8 if (a === 10) return true; // 10.0.0.0/8 if (a === 127) return true; // 回环 if (a === 169 && b === 254) return true; // 链路本地,含云元数据 if (a === 172 && b >= 16 && b <= 31) return true; // 172.16.0.0/12 if (a === 192 && b === 168) return true; // 192.168.0.0/16 if (a === 192 && b === 0) return true; // 192.0.0.0/24 与 192.0.2.0/24 if (a === 100 && b >= 64 && b <= 127) return true; // CGNAT 100.64.0.0/10 if (a === 198 && (b === 18 || b === 19)) return true; // 基准测试 198.18.0.0/15 if (a === 198 && b === 51) return true; // TEST-NET-2 if (a === 203 && b === 0) return true; // TEST-NET-3 if (a >= 224) return true; // 组播与保留段 return false; } function isBlockedIpv6(ip: string): boolean { // IPv4-mapped:::ffff:127.0.0.1 / ::ffff:7f00:1 const mapped = ip.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/); if (mapped) return isBlockedIpv4(mapped[1]); const groups = expandIpv6(ip); if (!groups) return true; // 无法解析,按不安全处理 const first = groups[0]; if (groups.every((g) => g === 0)) return true; // :: 未指定 if (groups.slice(0, 7).every((g) => g === 0) && groups[7] === 1) return true; // ::1 回环 if ((first & 0xfe00) === 0xfc00) return true; // fc00::/7 ULA if ((first & 0xffc0) === 0xfe80) return true; // fe80::/10 链路本地 if ((first & 0xff00) === 0xff00) return true; // ff00::/8 组播 // NAT64 64:ff9b::/96:内嵌 IPv4,取最后两组 if (groups[0] === 0x0064 && groups[1] === 0xff9b) { const embedded = [groups[6] >> 8, groups[6] & 0xff, groups[7] >> 8, groups[7] & 0xff].join("."); return isBlockedIpv4(embedded); } return false; } /** 把 IPv6 文本展开成 8 组 16 位整数;无法解析返回 null */ function expandIpv6(ip: string): number[] | null { let text = ip; // 内嵌 IPv4(如 ::ffff:1.2.3.4)先转成两组十六进制 const embedded = text.match(/^(.*:)(\d+\.\d+\.\d+\.\d+)$/); if (embedded) { const octets = embedded[2].split(".").map((p) => Number(p)); if (octets.some((n) => !Number.isInteger(n) || n < 0 || n > 255)) return null; const high = ((octets[0] << 8) | octets[1]).toString(16); const low = ((octets[2] << 8) | octets[3]).toString(16); text = `${embedded[1]}${high}:${low}`; } const halves = text.split("::"); if (halves.length > 2) return null; const parseGroups = (part: string): number[] | null => { if (!part) return []; const items = part.split(":"); const result: number[] = []; for (const item of items) { if (!/^[0-9a-f]{1,4}$/.test(item)) return null; result.push(parseInt(item, 16)); } return result; }; const head = parseGroups(halves[0]); const tail = halves.length === 2 ? parseGroups(halves[1]) : []; if (head === null || tail === null) return null; if (halves.length === 1) { return head.length === 8 ? head : null; } const fill = 8 - head.length - tail.length; if (fill < 0) return null; return [...head, ...new Array(fill).fill(0), ...tail]; } /** * 校验目标 URL 是否可以安全抓取。不通过时抛 {@link UnsafeUrlError}。 * * @param rawUrl 调用方给的 URL */ export async function assertSafeUrl(rawUrl: string): Promise { let url: URL; try { url = new URL(rawUrl); } catch { throw new UnsafeUrlError("无效的 URL"); } if (!ALLOWED_PROTOCOLS.has(url.protocol)) { throw new UnsafeUrlError("只支持抓取 http/https 链接"); } const port = url.port || (url.protocol === "https:" ? "443" : "80"); if (!allowedPorts().has(port)) { throw new UnsafeUrlError("这个链接的端口不在允许范围内"); } // 字面量就是 IP 时直接判定,避免走了 DNS 分支 const host = url.hostname.replace(/^\[|\]$/g, ""); if (isIpLiteral(host)) { if (isBlockedAddress(host)) { throw new UnsafeUrlError("不能抓取内网或本机地址"); } return url; } // 域名:解析后逐个校验,防止解析到内网或 DNS rebinding let addresses: string[]; try { const resolved = await lookup(host, { all: true }); addresses = resolved.map((item) => item.address); } catch { throw new UnsafeUrlError("无法解析这个链接的域名"); } if (addresses.length === 0) { throw new UnsafeUrlError("无法解析这个链接的域名"); } if (addresses.some((address) => isBlockedAddress(address))) { throw new UnsafeUrlError("不能抓取内网或本机地址"); } return url; } function isIpLiteral(host: string): boolean { if (/^\d+\.\d+\.\d+\.\d+$/.test(host)) return true; return host.includes(":"); } export interface SafeFetchResult { /** 最终跳转到的地址 */ finalUrl: string; /** 响应体(截断到上限) */ body: string; } /** * 带上限与逐跳校验的抓取。 * * @param rawUrl 目标地址 * @param timeoutMs 单次请求超时 */ export async function safeFetchText(rawUrl: string, timeoutMs = 10_000): Promise { let current = await assertSafeUrl(rawUrl); for (let hop = 0; hop <= MAX_REDIRECTS; hop++) { const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), timeoutMs); try { const response = await fetch(current, { signal: controller.signal, // 自己处理重定向,才能逐跳校验 redirect: "manual", headers: { accept: "text/html,application/xhtml+xml" }, }); if (isRedirect(response.status)) { const location = response.headers.get("location"); if (!location) { throw new UnsafeUrlError("重定向没有给出目标地址"); } // 相对跳转按当前地址解析,然后重新走一遍完整校验 current = await assertSafeUrl(new URL(location, current).toString()); continue; } if (!response.ok) { throw new UnsafeUrlError("目标地址没有正常返回内容"); } const body = await readCapped(response); return { finalUrl: current.toString(), body }; } finally { clearTimeout(timer); } } throw new UnsafeUrlError("跳转次数过多,已停止抓取"); } function isRedirect(status: number): boolean { return status === 301 || status === 302 || status === 303 || status === 307 || status === 308; } /** * 读取响应体并截断:既防超大响应打爆内存,也避免为一个标题读完整个页面。 */ async function readCapped(response: Response): Promise { const contentType = response.headers.get("content-type") ?? ""; if (contentType && !/text\/|xml|json/i.test(contentType)) { throw new UnsafeUrlError("这个链接不是网页内容"); } const body = response.body; if (!body) return ""; const reader = body.getReader(); const chunks: Uint8Array[] = []; let total = 0; try { for (;;) { const { done, value } = await reader.read(); if (done) break; if (!value) continue; chunks.push(value); total += value.byteLength; if (total >= MAX_RESPONSE_BYTES) break; } } finally { await reader.cancel().catch(() => undefined); } const buffer = Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))); const text = buffer.subarray(0, MAX_RESPONSE_BYTES).toString("utf-8"); return text; }