diff --git a/src/admin/routes.ts b/src/admin/routes.ts index fbf93f2..5d2f9eb 100644 --- a/src/admin/routes.ts +++ b/src/admin/routes.ts @@ -4,6 +4,7 @@ import type { FastifyInstance } from "fastify"; import { logStore } from "./store.js"; +import { checkToken, configuredToken } from "../utils/auth.js"; const PANEL_HTML = ` @@ -238,6 +239,19 @@ init(); export async function adminRoutes(app: FastifyInstance): Promise { + // 管理面板会展示完整 prompt 与学习内容,与 /ai/* 共用同一个可选令牌(安全-03) + if (!configuredToken()) { + app.log.warn( + "未设置 LPT_AI_TOKEN,/admin 面板与 /ai/* 无访问保护(面板会展示 prompt 与学习内容)", + ); + } + + app.addHook("preHandler", async (request, reply) => { + if (!checkToken(request, reply)) { + return reply; + } + }); + // —— 面板页面 —— app.get("/admin", async (_request, reply) => { return reply.type("text/html; charset=utf-8").send(PANEL_HTML); diff --git a/src/index.ts b/src/index.ts index 807f844..225627e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -28,6 +28,9 @@ const app = Fastify({ redact: ["req.headers.authorization"], // 永不记录密钥 }, connectionTimeout: 180_000, // 3 min + // 安全-03:请求体上限。抓取标题与 AI 任务参数都是小报文, + // 没有上限时超大 body 可以打爆内存。 + bodyLimit: Number(process.env.LPT_AI_BODY_LIMIT_BYTES ?? 256 * 1024), }); await app.register(aiRoutes); diff --git a/src/routes/ai.ts b/src/routes/ai.ts index 3e6032b..5dfc1f2 100644 --- a/src/routes/ai.ts +++ b/src/routes/ai.ts @@ -6,10 +6,37 @@ import type { FastifyInstance } from "fastify"; import { isAvailable, loadConfig } from "../llm/client.js"; import { createTask, getTask, type TaskType } from "../task-queue.js"; import { logStore } from "../admin/store.js"; +import { safeFetchText, UnsafeUrlError } from "../utils/ssrf-guard.js"; +import { RateLimiter, intFromEnv } from "../utils/rate-limit.js"; +import { checkToken } from "../utils/auth.js"; export async function aiRoutes(app: FastifyInstance): Promise { const config = loadConfig(); + // 安全-03:AI 任务会消耗 LLM 额度,按来源限流(单进程粗粒度保护) + const taskLimiter = new RateLimiter({ + limit: intFromEnv("LPT_AI_RATE_LIMIT", 30), + windowMs: intFromEnv("LPT_AI_RATE_WINDOW_MS", 60_000), + }); + + // 安全-03:可选的入站令牌。设置了 LPT_AI_TOKEN 时,所有 /ai/* 请求都必须携带 + app.addHook("preHandler", async (request, reply) => { + if (!request.url.startsWith("/ai/")) return; + if (!checkToken(request, reply)) { + return reply; + } + return undefined; + }); + + /** 取限流用的来源标识:优先真实客户端 IP(后面有反代时取 X-Forwarded-For 首段) */ + const clientKey = (request: { ip?: string; headers: Record }): string => { + const forwarded = request.headers["x-forwarded-for"]; + if (typeof forwarded === "string" && forwarded.trim()) { + return forwarded.split(",")[0].trim(); + } + return request.ip ?? "unknown"; + }; + app.get("/health", async () => ({ status: "ok", llmAvailable: isAvailable(config), @@ -29,6 +56,15 @@ export async function aiRoutes(app: FastifyInstance): Promise { }>("/ai/tasks", async (request, reply) => { const { type, params, idempotencyKey, force } = request.body ?? {}; + const decision = taskLimiter.check(clientKey(request)); + if (!decision.allowed) { + app.log.warn({ key: clientKey(request) }, "AI 任务提交被限流"); + return reply + .status(429) + .header("retry-after", String(Math.ceil(decision.retryAfterMs / 1000))) + .send({ error: "提交太频繁了,请稍后再试" }); + } + if (!type || !["aggregate-report", "generate-mind-map", "compare-recall"].includes(type)) { return reply.status(400).send({ error: "无效的 type,允许: aggregate-report, generate-mind-map, compare-recall" }); } @@ -58,23 +94,23 @@ export async function aiRoutes(app: FastifyInstance): Promise { return task; }); - /** 抓取网页标题 */ + /** + * 抓取网页标题。 + * + *

安全-03:目标由调用方给出,因此在抓取前做协议、端口、DNS 解析与逐跳重定向校验, + * 避免被当成探测内网与云元数据的跳板,同时对响应体做截断。

+ */ app.post<{ Body: { url: string } }>("/fetch-title", async (request, reply) => { const { url } = request.body ?? {}; const t0 = Date.now(); - if (!url || typeof url !== "string" || (!url.startsWith("http://") && !url.startsWith("https://"))) { + if (!url || typeof url !== "string") { return reply.status(400).send({ error: "无效的 URL" }); } try { - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), 10_000); - const response = await fetch(url, { signal: controller.signal }); - clearTimeout(timer); - - const html = await response.text(); - const match = html.match(/]*>([^<]+)<\/title>/i); + const { body } = await safeFetchText(url); + const match = body.match(/]*>([^<]+)<\/title>/i); const title = match ? match[1].trim() : null; logStore.add({ @@ -88,16 +124,22 @@ export async function aiRoutes(app: FastifyInstance): Promise { }); return { url, title }; } catch (err) { + // 被安全校验拦下时返回 400(调用方的问题),其余按抓取失败处理 + const blocked = err instanceof UnsafeUrlError; + const message = err instanceof Error ? err.message : String(err); logStore.add({ timestamp: new Date().toISOString(), endpoint: "/fetch-title", model: "", durationMs: Date.now() - t0, - status: 502, + status: blocked ? 400 : 502, requestBody: { url }, responseBody: { url, title: null }, - error: err instanceof Error ? err.message : String(err), + error: message, }); + if (blocked) { + return reply.status(400).send({ error: message }); + } return { url, title: null }; } }); diff --git a/src/utils/auth.test.ts b/src/utils/auth.test.ts new file mode 100644 index 0000000..df72c0a --- /dev/null +++ b/src/utils/auth.test.ts @@ -0,0 +1,73 @@ +import { afterEach, describe, expect, it } from "vitest"; + +import { checkToken, configuredToken, TOKEN_HEADER } from "./auth.js"; + +/** 极简 reply 替身:记录是否写出 401 */ +const makeReply = () => { + const calls: number[] = []; + return { + calls, + status(code: number) { + calls.push(code); + return { send: (body: unknown) => body }; + }, + }; +}; + +const original = process.env.LPT_AI_TOKEN; + +afterEach(() => { + if (original === undefined) { + delete process.env.LPT_AI_TOKEN; + } else { + process.env.LPT_AI_TOKEN = original; + } +}); + +describe("AI 入站令牌", () => { + it("未配置令牌时放行,保持开发环境零摩擦", () => { + delete process.env.LPT_AI_TOKEN; + expect(configuredToken()).toBe(""); + + const reply = makeReply(); + expect(checkToken({ headers: {} }, reply)).toBe(true); + expect(reply.calls).toEqual([]); + }); + + it("配置令牌后,缺令牌的请求被拒绝", () => { + process.env.LPT_AI_TOKEN = "secret-token"; + const reply = makeReply(); + + expect(checkToken({ headers: {} }, reply)).toBe(false); + expect(reply.calls).toEqual([401]); + }); + + it("配置令牌后,header 提供正确令牌放行", () => { + process.env.LPT_AI_TOKEN = "secret-token"; + const reply = makeReply(); + + expect(checkToken({ headers: { [TOKEN_HEADER]: "secret-token" } }, reply)).toBe(true); + expect(reply.calls).toEqual([]); + }); + + it("支持用 query 传令牌(便于在浏览器里打开 /admin)", () => { + process.env.LPT_AI_TOKEN = "secret-token"; + const reply = makeReply(); + + expect(checkToken({ headers: {}, query: { token: "secret-token" } }, reply)).toBe(true); + }); + + it("令牌错误被拒绝", () => { + process.env.LPT_AI_TOKEN = "secret-token"; + const reply = makeReply(); + + expect(checkToken({ headers: { [TOKEN_HEADER]: "wrong" } }, reply)).toBe(false); + expect(reply.calls).toEqual([401]); + }); + + it("空白令牌视为未配置", () => { + process.env.LPT_AI_TOKEN = " "; + expect(configuredToken()).toBe(""); + expect(checkToken({ headers: {} }, makeReply())).toBe(true); + }); +}); diff --git a/src/utils/auth.ts b/src/utils/auth.ts new file mode 100644 index 0000000..57dfade --- /dev/null +++ b/src/utils/auth.ts @@ -0,0 +1,43 @@ +/** + * 入站访问令牌(安全-03)。 + * + * 设计取舍:令牌是**可选**的。 + * - 设置 `LPT_AI_TOKEN` 后,`/ai/*` 与 `/admin` 都要求携带令牌(header `x-api-token` 或 query `token`); + * - 未设置时保持可访问,并在启动日志里明确告知,避免开发阶段被迫维护额外凭据。 + * + * 生产环境只需往 `lpt-secrets` 里加一个 key 即可收紧(部署清单已用 `optional: true` 引用, + * 不设置也不会导致 Pod 起不来)。 + */ + +export const TOKEN_HEADER = "x-api-token"; + +/** 当前配置的令牌;空字符串表示未启用 */ +export function configuredToken(): string { + return (process.env.LPT_AI_TOKEN ?? "").trim(); +} + +interface TokenRequest { + headers: Record; + query?: unknown; +} + +/** + * 校验请求是否携带了正确的令牌。 + * + * @returns true 表示放行;false 表示已写出 401 响应,调用方应直接结束处理 + */ +export function checkToken( + request: TokenRequest, + reply: { status: (code: number) => { send: (body: unknown) => unknown } }, +): boolean { + const token = configuredToken(); + if (!token) return true; + + const header = request.headers[TOKEN_HEADER]; + const query = request.query as { token?: string } | undefined; + const provided = typeof header === "string" ? header : query?.token; + if (provided === token) return true; + + reply.status(401).send({ error: "未授权" }); + return false; +} diff --git a/src/utils/rate-limit.test.ts b/src/utils/rate-limit.test.ts new file mode 100644 index 0000000..efdfbdc --- /dev/null +++ b/src/utils/rate-limit.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from "vitest"; + +import { RateLimiter, intFromEnv } from "./rate-limit.js"; + +describe("RateLimiter", () => { + it("窗口内放行到上限,超出后拒绝并给出等待时间", () => { + const limiter = new RateLimiter({ limit: 3, windowMs: 1000 }); + const t0 = 1_000_000; + + expect(limiter.check("a", t0).allowed).toBe(true); + expect(limiter.check("a", t0 + 10).allowed).toBe(true); + expect(limiter.check("a", t0 + 20).allowed).toBe(true); + + const blocked = limiter.check("a", t0 + 30); + expect(blocked.allowed).toBe(false); + expect(blocked.retryAfterMs).toBeGreaterThan(0); + expect(blocked.retryAfterMs).toBeLessThanOrEqual(1000); + }); + + it("窗口滑过之后重新放行", () => { + const limiter = new RateLimiter({ limit: 1, windowMs: 1000 }); + const t0 = 1_000_000; + + expect(limiter.check("a", t0).allowed).toBe(true); + expect(limiter.check("a", t0 + 500).allowed).toBe(false); + expect(limiter.check("a", t0 + 1001).allowed).toBe(true); + }); + + it("不同来源互不影响", () => { + const limiter = new RateLimiter({ limit: 1, windowMs: 1000 }); + const t0 = 1_000_000; + + expect(limiter.check("a", t0).allowed).toBe(true); + expect(limiter.check("b", t0).allowed).toBe(true); + expect(limiter.check("a", t0).allowed).toBe(false); + }); + + it("清理过期来源,避免 Map 无界增长", () => { + const limiter = new RateLimiter({ limit: 1, windowMs: 1000 }); + const t0 = 1_000_000; + + limiter.check("a", t0); + limiter.check("b", t0); + // 跨过多个窗口后再访问,触发清理 + limiter.check("c", t0 + 5000); + + const internal = limiter as unknown as { hits: Map }; + expect(internal.hits.has("a")).toBe(false); + expect(internal.hits.has("b")).toBe(false); + expect(internal.hits.has("c")).toBe(true); + }); +}); + +describe("intFromEnv", () => { + it("读取正整数,非法值回退默认", () => { + process.env.TEST_INT_ENV = "42"; + expect(intFromEnv("TEST_INT_ENV", 7)).toBe(42); + + process.env.TEST_INT_ENV = "abc"; + expect(intFromEnv("TEST_INT_ENV", 7)).toBe(7); + + process.env.TEST_INT_ENV = "-1"; + expect(intFromEnv("TEST_INT_ENV", 7)).toBe(7); + + delete process.env.TEST_INT_ENV; + expect(intFromEnv("TEST_INT_ENV", 7)).toBe(7); + }); +}); diff --git a/src/utils/rate-limit.ts b/src/utils/rate-limit.ts new file mode 100644 index 0000000..89b10db --- /dev/null +++ b/src/utils/rate-limit.ts @@ -0,0 +1,68 @@ +/** + * 进程内限流(安全-03)。 + * + * 目的:`/ai/tasks` 每次调用都会消耗 LLM 额度,需要防止误用刷爆。 + * 不引入额外依赖,按「每窗口每来源」计数即可。 + * + * 注意:这是单进程内存计数。多副本部署时每个副本各算一份,属于粗粒度保护, + * 不是精确配额;真要做精确配额需要放到共享存储(当前不需要)。 + */ + +export interface RateLimitOptions { + /** 窗口内允许的请求数 */ + limit: number; + /** 窗口长度(毫秒) */ + windowMs: number; +} + +export interface RateLimitDecision { + allowed: boolean; + /** 建议告诉调用方还要等多久(毫秒) */ + retryAfterMs: number; +} + +export class RateLimiter { + private readonly hits = new Map(); + private lastSweep = 0; + + constructor(private readonly options: RateLimitOptions) {} + + /** 记一次命中并返回是否放行 */ + check(key: string, now = Date.now()): RateLimitDecision { + this.sweep(now); + + const windowStart = now - this.options.windowMs; + const recent = (this.hits.get(key) ?? []).filter((at) => at > windowStart); + if (recent.length >= this.options.limit) { + this.hits.set(key, recent); + const oldest = recent[0]; + return { allowed: false, retryAfterMs: Math.max(0, oldest + this.options.windowMs - now) }; + } + recent.push(now); + this.hits.set(key, recent); + return { allowed: true, retryAfterMs: 0 }; + } + + /** 清理过期的 key,避免长期运行后 Map 无界增长 */ + private sweep(now: number): void { + if (now - this.lastSweep < this.options.windowMs) return; + this.lastSweep = now; + const windowStart = now - this.options.windowMs; + for (const [key, times] of this.hits) { + const recent = times.filter((at) => at > windowStart); + if (recent.length === 0) { + this.hits.delete(key); + } else { + this.hits.set(key, recent); + } + } + } +} + +/** 从环境变量读一个正整数,非法时返回默认值 */ +export function intFromEnv(name: string, fallback: number): number { + const raw = process.env[name]; + if (!raw) return fallback; + const value = Number(raw); + return Number.isInteger(value) && value > 0 ? value : fallback; +} diff --git a/src/utils/ssrf-guard.test.ts b/src/utils/ssrf-guard.test.ts new file mode 100644 index 0000000..5a703fb --- /dev/null +++ b/src/utils/ssrf-guard.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it } from "vitest"; + +import { isBlockedAddress, UnsafeUrlError, assertSafeUrl } from "./ssrf-guard.js"; + +describe("isBlockedAddress — IPv4", () => { + it("拦下私网、回环、链路本地(含云元数据)与保留段", () => { + const blocked = [ + "127.0.0.1", + "127.255.255.254", + "10.0.0.1", + "10.255.255.255", + "172.16.0.1", + "172.31.255.255", + "192.168.1.1", + "169.254.169.254", // 云元数据 + "0.0.0.0", + "100.64.0.1", // CGNAT + "224.0.0.1", // 组播 + "255.255.255.255", + ]; + for (const ip of blocked) { + expect(isBlockedAddress(ip), ip).toBe(true); + } + }); + + it("放行公网地址", () => { + const allowed = ["8.8.8.8", "1.1.1.1", "100.63.255.255", "172.32.0.1", "192.169.0.1", "223.255.255.255"]; + for (const ip of allowed) { + expect(isBlockedAddress(ip), ip).toBe(false); + } + }); + + it("非法 IPv4 字面量按不安全处理", () => { + expect(isBlockedAddress("999.1.1.1")).toBe(true); + expect(isBlockedAddress("1.2.3")).toBe(true); + expect(isBlockedAddress("")).toBe(true); + }); +}); + +describe("isBlockedAddress — IPv6", () => { + it("拦下回环、未指定、ULA、链路本地与组播", () => { + const blocked = ["::1", "::", "fc00::1", "fd12:3456::1", "fe80::1", "ff02::1", "fe80::1%eth0"]; + for (const ip of blocked) { + expect(isBlockedAddress(ip), ip).toBe(true); + } + }); + + it("识别 IPv4-mapped 内嵌的私网地址(绕过尝试)", () => { + expect(isBlockedAddress("::ffff:127.0.0.1")).toBe(true); + expect(isBlockedAddress("::ffff:169.254.169.254")).toBe(true); + expect(isBlockedAddress("::ffff:8.8.8.8")).toBe(false); + }); + + it("识别 NAT64 内嵌的私网地址", () => { + expect(isBlockedAddress("64:ff9b::7f00:1")).toBe(true); // 内嵌 127.0.0.1 + expect(isBlockedAddress("64:ff9b::808:808")).toBe(false); // 内嵌 8.8.8.8 + }); + + it("放行公网 IPv6", () => { + expect(isBlockedAddress("2001:4860:4860::8888")).toBe(false); + }); +}); + +describe("assertSafeUrl", () => { + it("只允许 http/https", async () => { + await expect(assertSafeUrl("file:///etc/passwd")).rejects.toBeInstanceOf(UnsafeUrlError); + await expect(assertSafeUrl("ftp://example.com")).rejects.toBeInstanceOf(UnsafeUrlError); + await expect(assertSafeUrl("gopher://example.com")).rejects.toBeInstanceOf(UnsafeUrlError); + }); + + it("拒绝无效 URL", async () => { + await expect(assertSafeUrl("not a url")).rejects.toBeInstanceOf(UnsafeUrlError); + }); + + it("拒绝内网 IP 字面量,不经过 DNS", async () => { + await expect(assertSafeUrl("http://127.0.0.1:80/")).rejects.toBeInstanceOf(UnsafeUrlError); + await expect(assertSafeUrl("http://192.168.1.1/")).rejects.toBeInstanceOf(UnsafeUrlError); + await expect(assertSafeUrl("http://169.254.169.254/latest/meta-data/")).rejects.toBeInstanceOf( + UnsafeUrlError, + ); + await expect(assertSafeUrl("http://[::1]/")).rejects.toBeInstanceOf(UnsafeUrlError); + }); + + it("拒绝非 http/https 默认端口的地址(避免探测内网服务)", async () => { + await expect(assertSafeUrl("http://8.8.8.8:6379/")).rejects.toBeInstanceOf(UnsafeUrlError); + await expect(assertSafeUrl("http://8.8.8.8:3306/")).rejects.toBeInstanceOf(UnsafeUrlError); + }); + + it("放行公网 http/https 字面量与默认端口", async () => { + await expect(assertSafeUrl("http://8.8.8.8/")).resolves.toBeInstanceOf(URL); + await expect(assertSafeUrl("https://8.8.8.8:443/")).resolves.toBeInstanceOf(URL); + }); + + it("域名解析失败时拒绝", async () => { + await expect(assertSafeUrl("https://this-domain-should-not-resolve.invalid/")).rejects.toBeInstanceOf( + UnsafeUrlError, + ); + }); + + it("localhost 这类解析到回环的域名被拦下(防绕过字面量检查)", async () => { + await expect(assertSafeUrl("http://localhost:80/")).rejects.toBeInstanceOf(UnsafeUrlError); + }); +}); diff --git a/src/utils/ssrf-guard.ts b/src/utils/ssrf-guard.ts new file mode 100644 index 0000000..5621427 --- /dev/null +++ b/src/utils/ssrf-guard.ts @@ -0,0 +1,285 @@ +/** + * SSRF 防护:校验「服务端主动抓取」的目标地址是否安全。 + * + * 背景:`/fetch-title` 由服务端抓取调用方给的 URL。如果不做限制,调用方可以拿它探测 + * 内网服务与云元数据地址(169.254.169.254),把本服务当成内网跳板。 + * + * 防护分四层,缺一层都能绕过: + * 1. 协议只允许 http/https; + * 2. 端口只允许 http/https 默认端口(外加显式配置的额外端口),避免被用来探测 Redis/MySQL 等内网服务; + * 3. 解析域名后校验**每一个**解析结果,而不是只校验字面量(防 DNS 解析到内网、防 DNS rebinding); + * 4. 重定向逐跳校验,避免「首跳合法、次跳内网」。 + */ + +import { lookup } from "node:dns/promises"; + +/** 允许的协议 */ +const ALLOWED_PROTOCOLS = new Set(["http:", "https:"]); + +/** 默认允许的端口 */ +const DEFAULT_ALLOWED_PORTS = new Set(["80", "443"]); + +/** 响应体读取上限(字节):标题通常在头部,不必读完整个页面 */ +export const MAX_RESPONSE_BYTES = 512 * 1024; + +/** 最大重定向跳数 */ +export const MAX_REDIRECTS = 3; + +export class UnsafeUrlError extends Error { + constructor(message: string) { + super(message); + this.name = "UnsafeUrlError"; + } +} + +/** 解析额外允许的端口:环境变量 LPT_AI_FETCH_ALLOWED_PORTS,逗号分隔 */ +function allowedPorts(): Set { + const extra = (process.env.LPT_AI_FETCH_ALLOWED_PORTS ?? "") + .split(",") + .map((p) => p.trim()) + .filter((p) => /^\d+$/.test(p)); + return new Set([...DEFAULT_ALLOWED_PORTS, ...extra]); +} + +/** + * 判断一个 IP 字面量是否属于「不可主动访问」的地址段。 + * + * 覆盖 IPv4 私网/回环/链路本地(含云元数据 169.254.169.254)/CGNAT/组播/保留段, + * 以及 IPv6 回环、未指定、ULA、链路本地、组播,并识别 IPv4-mapped 与 NAT64 里内嵌的 IPv4。 + */ +export function isBlockedAddress(address: string): boolean { + const ip = address.trim().toLowerCase(); + if (!ip) return true; + + // IPv6:去掉可能的 zone id(fe80::1%eth0) + const bare = ip.includes("%") ? ip.slice(0, ip.indexOf("%")) : ip; + + if (bare.includes(":")) { + return isBlockedIpv6(bare); + } + return isBlockedIpv4(bare); +} + +function isBlockedIpv4(ip: string): boolean { + const parts = ip.split("."); + if (parts.length !== 4) return true; // 不是合法 IPv4,按不安全处理 + const octets = parts.map((p) => Number(p)); + if (octets.some((n) => !Number.isInteger(n) || n < 0 || n > 255)) return true; + + const [a, b] = octets; + if (a === 0) return true; // 0.0.0.0/8 + if (a === 10) return true; // 10.0.0.0/8 + if (a === 127) return true; // 回环 + if (a === 169 && b === 254) return true; // 链路本地,含云元数据 + if (a === 172 && b >= 16 && b <= 31) return true; // 172.16.0.0/12 + if (a === 192 && b === 168) return true; // 192.168.0.0/16 + if (a === 192 && b === 0) return true; // 192.0.0.0/24 与 192.0.2.0/24 + if (a === 100 && b >= 64 && b <= 127) return true; // CGNAT 100.64.0.0/10 + if (a === 198 && (b === 18 || b === 19)) return true; // 基准测试 198.18.0.0/15 + if (a === 198 && b === 51) return true; // TEST-NET-2 + if (a === 203 && b === 0) return true; // TEST-NET-3 + if (a >= 224) return true; // 组播与保留段 + return false; +} + +function isBlockedIpv6(ip: string): boolean { + // IPv4-mapped:::ffff:127.0.0.1 / ::ffff:7f00:1 + const mapped = ip.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/); + if (mapped) return isBlockedIpv4(mapped[1]); + + const groups = expandIpv6(ip); + if (!groups) return true; // 无法解析,按不安全处理 + + const first = groups[0]; + if (groups.every((g) => g === 0)) return true; // :: 未指定 + if (groups.slice(0, 7).every((g) => g === 0) && groups[7] === 1) return true; // ::1 回环 + if ((first & 0xfe00) === 0xfc00) return true; // fc00::/7 ULA + if ((first & 0xffc0) === 0xfe80) return true; // fe80::/10 链路本地 + if ((first & 0xff00) === 0xff00) return true; // ff00::/8 组播 + + // NAT64 64:ff9b::/96:内嵌 IPv4,取最后两组 + if (groups[0] === 0x0064 && groups[1] === 0xff9b) { + const embedded = [groups[6] >> 8, groups[6] & 0xff, groups[7] >> 8, groups[7] & 0xff].join("."); + return isBlockedIpv4(embedded); + } + return false; +} + +/** 把 IPv6 文本展开成 8 组 16 位整数;无法解析返回 null */ +function expandIpv6(ip: string): number[] | null { + let text = ip; + // 内嵌 IPv4(如 ::ffff:1.2.3.4)先转成两组十六进制 + const embedded = text.match(/^(.*:)(\d+\.\d+\.\d+\.\d+)$/); + if (embedded) { + const octets = embedded[2].split(".").map((p) => Number(p)); + if (octets.some((n) => !Number.isInteger(n) || n < 0 || n > 255)) return null; + const high = ((octets[0] << 8) | octets[1]).toString(16); + const low = ((octets[2] << 8) | octets[3]).toString(16); + text = `${embedded[1]}${high}:${low}`; + } + + const halves = text.split("::"); + if (halves.length > 2) return null; + + const parseGroups = (part: string): number[] | null => { + if (!part) return []; + const items = part.split(":"); + const result: number[] = []; + for (const item of items) { + if (!/^[0-9a-f]{1,4}$/.test(item)) return null; + result.push(parseInt(item, 16)); + } + return result; + }; + + const head = parseGroups(halves[0]); + const tail = halves.length === 2 ? parseGroups(halves[1]) : []; + if (head === null || tail === null) return null; + + if (halves.length === 1) { + return head.length === 8 ? head : null; + } + const fill = 8 - head.length - tail.length; + if (fill < 0) return null; + return [...head, ...new Array(fill).fill(0), ...tail]; +} + +/** + * 校验目标 URL 是否可以安全抓取。不通过时抛 {@link UnsafeUrlError}。 + * + * @param rawUrl 调用方给的 URL + */ +export async function assertSafeUrl(rawUrl: string): Promise { + let url: URL; + try { + url = new URL(rawUrl); + } catch { + throw new UnsafeUrlError("无效的 URL"); + } + + if (!ALLOWED_PROTOCOLS.has(url.protocol)) { + throw new UnsafeUrlError("只支持抓取 http/https 链接"); + } + + const port = url.port || (url.protocol === "https:" ? "443" : "80"); + if (!allowedPorts().has(port)) { + throw new UnsafeUrlError("这个链接的端口不在允许范围内"); + } + + // 字面量就是 IP 时直接判定,避免走了 DNS 分支 + const host = url.hostname.replace(/^\[|\]$/g, ""); + if (isIpLiteral(host)) { + if (isBlockedAddress(host)) { + throw new UnsafeUrlError("不能抓取内网或本机地址"); + } + return url; + } + + // 域名:解析后逐个校验,防止解析到内网或 DNS rebinding + let addresses: string[]; + try { + const resolved = await lookup(host, { all: true }); + addresses = resolved.map((item) => item.address); + } catch { + throw new UnsafeUrlError("无法解析这个链接的域名"); + } + if (addresses.length === 0) { + throw new UnsafeUrlError("无法解析这个链接的域名"); + } + if (addresses.some((address) => isBlockedAddress(address))) { + throw new UnsafeUrlError("不能抓取内网或本机地址"); + } + return url; +} + +function isIpLiteral(host: string): boolean { + if (/^\d+\.\d+\.\d+\.\d+$/.test(host)) return true; + return host.includes(":"); +} + +export interface SafeFetchResult { + /** 最终跳转到的地址 */ + finalUrl: string; + /** 响应体(截断到上限) */ + body: string; +} + +/** + * 带上限与逐跳校验的抓取。 + * + * @param rawUrl 目标地址 + * @param timeoutMs 单次请求超时 + */ +export async function safeFetchText(rawUrl: string, timeoutMs = 10_000): Promise { + let current = await assertSafeUrl(rawUrl); + + for (let hop = 0; hop <= MAX_REDIRECTS; hop++) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetch(current, { + signal: controller.signal, + // 自己处理重定向,才能逐跳校验 + redirect: "manual", + headers: { accept: "text/html,application/xhtml+xml" }, + }); + + if (isRedirect(response.status)) { + const location = response.headers.get("location"); + if (!location) { + throw new UnsafeUrlError("重定向没有给出目标地址"); + } + // 相对跳转按当前地址解析,然后重新走一遍完整校验 + current = await assertSafeUrl(new URL(location, current).toString()); + continue; + } + + if (!response.ok) { + throw new UnsafeUrlError("目标地址没有正常返回内容"); + } + + const body = await readCapped(response); + return { finalUrl: current.toString(), body }; + } finally { + clearTimeout(timer); + } + } + throw new UnsafeUrlError("跳转次数过多,已停止抓取"); +} + +function isRedirect(status: number): boolean { + return status === 301 || status === 302 || status === 303 || status === 307 || status === 308; +} + +/** + * 读取响应体并截断:既防超大响应打爆内存,也避免为一个标题读完整个页面。 + */ +async function readCapped(response: Response): Promise { + const contentType = response.headers.get("content-type") ?? ""; + if (contentType && !/text\/|xml|json/i.test(contentType)) { + throw new UnsafeUrlError("这个链接不是网页内容"); + } + + const body = response.body; + if (!body) return ""; + + const reader = body.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + if (!value) continue; + chunks.push(value); + total += value.byteLength; + if (total >= MAX_RESPONSE_BYTES) break; + } + } finally { + await reader.cancel().catch(() => undefined); + } + + const buffer = Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))); + const text = buffer.subarray(0, MAX_RESPONSE_BYTES).toString("utf-8"); + return text; +}