feat: 加固标题抓取并加上限流与可选入站令牌
This commit is contained in:
@@ -0,0 +1,73 @@
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
|
||||
import { checkToken, configuredToken, TOKEN_HEADER } from "./auth.js";
|
||||
|
||||
/** 极简 reply 替身:记录是否写出 401 */
|
||||
const makeReply = () => {
|
||||
const calls: number[] = [];
|
||||
return {
|
||||
calls,
|
||||
status(code: number) {
|
||||
calls.push(code);
|
||||
return { send: (body: unknown) => body };
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
const original = process.env.LPT_AI_TOKEN;
|
||||
|
||||
afterEach(() => {
|
||||
if (original === undefined) {
|
||||
delete process.env.LPT_AI_TOKEN;
|
||||
} else {
|
||||
process.env.LPT_AI_TOKEN = original;
|
||||
}
|
||||
});
|
||||
|
||||
describe("AI 入站令牌", () => {
|
||||
it("未配置令牌时放行,保持开发环境零摩擦", () => {
|
||||
delete process.env.LPT_AI_TOKEN;
|
||||
expect(configuredToken()).toBe("");
|
||||
|
||||
const reply = makeReply();
|
||||
expect(checkToken({ headers: {} }, reply)).toBe(true);
|
||||
expect(reply.calls).toEqual([]);
|
||||
});
|
||||
|
||||
it("配置令牌后,缺令牌的请求被拒绝", () => {
|
||||
process.env.LPT_AI_TOKEN = "secret-token";
|
||||
const reply = makeReply();
|
||||
|
||||
expect(checkToken({ headers: {} }, reply)).toBe(false);
|
||||
expect(reply.calls).toEqual([401]);
|
||||
});
|
||||
|
||||
it("配置令牌后,header 提供正确令牌放行", () => {
|
||||
process.env.LPT_AI_TOKEN = "secret-token";
|
||||
const reply = makeReply();
|
||||
|
||||
expect(checkToken({ headers: { [TOKEN_HEADER]: "secret-token" } }, reply)).toBe(true);
|
||||
expect(reply.calls).toEqual([]);
|
||||
});
|
||||
|
||||
it("支持用 query 传令牌(便于在浏览器里打开 /admin)", () => {
|
||||
process.env.LPT_AI_TOKEN = "secret-token";
|
||||
const reply = makeReply();
|
||||
|
||||
expect(checkToken({ headers: {}, query: { token: "secret-token" } }, reply)).toBe(true);
|
||||
});
|
||||
|
||||
it("令牌错误被拒绝", () => {
|
||||
process.env.LPT_AI_TOKEN = "secret-token";
|
||||
const reply = makeReply();
|
||||
|
||||
expect(checkToken({ headers: { [TOKEN_HEADER]: "wrong" } }, reply)).toBe(false);
|
||||
expect(reply.calls).toEqual([401]);
|
||||
});
|
||||
|
||||
it("空白令牌视为未配置", () => {
|
||||
process.env.LPT_AI_TOKEN = " ";
|
||||
expect(configuredToken()).toBe("");
|
||||
expect(checkToken({ headers: {} }, makeReply())).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,43 @@
|
||||
/**
|
||||
* 入站访问令牌(安全-03)。
|
||||
*
|
||||
* 设计取舍:令牌是**可选**的。
|
||||
* - 设置 `LPT_AI_TOKEN` 后,`/ai/*` 与 `/admin` 都要求携带令牌(header `x-api-token` 或 query `token`);
|
||||
* - 未设置时保持可访问,并在启动日志里明确告知,避免开发阶段被迫维护额外凭据。
|
||||
*
|
||||
* 生产环境只需往 `lpt-secrets` 里加一个 key 即可收紧(部署清单已用 `optional: true` 引用,
|
||||
* 不设置也不会导致 Pod 起不来)。
|
||||
*/
|
||||
|
||||
export const TOKEN_HEADER = "x-api-token";
|
||||
|
||||
/** 当前配置的令牌;空字符串表示未启用 */
|
||||
export function configuredToken(): string {
|
||||
return (process.env.LPT_AI_TOKEN ?? "").trim();
|
||||
}
|
||||
|
||||
interface TokenRequest {
|
||||
headers: Record<string, unknown>;
|
||||
query?: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验请求是否携带了正确的令牌。
|
||||
*
|
||||
* @returns true 表示放行;false 表示已写出 401 响应,调用方应直接结束处理
|
||||
*/
|
||||
export function checkToken(
|
||||
request: TokenRequest,
|
||||
reply: { status: (code: number) => { send: (body: unknown) => unknown } },
|
||||
): boolean {
|
||||
const token = configuredToken();
|
||||
if (!token) return true;
|
||||
|
||||
const header = request.headers[TOKEN_HEADER];
|
||||
const query = request.query as { token?: string } | undefined;
|
||||
const provided = typeof header === "string" ? header : query?.token;
|
||||
if (provided === token) return true;
|
||||
|
||||
reply.status(401).send({ error: "未授权" });
|
||||
return false;
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { RateLimiter, intFromEnv } from "./rate-limit.js";
|
||||
|
||||
describe("RateLimiter", () => {
|
||||
it("窗口内放行到上限,超出后拒绝并给出等待时间", () => {
|
||||
const limiter = new RateLimiter({ limit: 3, windowMs: 1000 });
|
||||
const t0 = 1_000_000;
|
||||
|
||||
expect(limiter.check("a", t0).allowed).toBe(true);
|
||||
expect(limiter.check("a", t0 + 10).allowed).toBe(true);
|
||||
expect(limiter.check("a", t0 + 20).allowed).toBe(true);
|
||||
|
||||
const blocked = limiter.check("a", t0 + 30);
|
||||
expect(blocked.allowed).toBe(false);
|
||||
expect(blocked.retryAfterMs).toBeGreaterThan(0);
|
||||
expect(blocked.retryAfterMs).toBeLessThanOrEqual(1000);
|
||||
});
|
||||
|
||||
it("窗口滑过之后重新放行", () => {
|
||||
const limiter = new RateLimiter({ limit: 1, windowMs: 1000 });
|
||||
const t0 = 1_000_000;
|
||||
|
||||
expect(limiter.check("a", t0).allowed).toBe(true);
|
||||
expect(limiter.check("a", t0 + 500).allowed).toBe(false);
|
||||
expect(limiter.check("a", t0 + 1001).allowed).toBe(true);
|
||||
});
|
||||
|
||||
it("不同来源互不影响", () => {
|
||||
const limiter = new RateLimiter({ limit: 1, windowMs: 1000 });
|
||||
const t0 = 1_000_000;
|
||||
|
||||
expect(limiter.check("a", t0).allowed).toBe(true);
|
||||
expect(limiter.check("b", t0).allowed).toBe(true);
|
||||
expect(limiter.check("a", t0).allowed).toBe(false);
|
||||
});
|
||||
|
||||
it("清理过期来源,避免 Map 无界增长", () => {
|
||||
const limiter = new RateLimiter({ limit: 1, windowMs: 1000 });
|
||||
const t0 = 1_000_000;
|
||||
|
||||
limiter.check("a", t0);
|
||||
limiter.check("b", t0);
|
||||
// 跨过多个窗口后再访问,触发清理
|
||||
limiter.check("c", t0 + 5000);
|
||||
|
||||
const internal = limiter as unknown as { hits: Map<string, number[]> };
|
||||
expect(internal.hits.has("a")).toBe(false);
|
||||
expect(internal.hits.has("b")).toBe(false);
|
||||
expect(internal.hits.has("c")).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("intFromEnv", () => {
|
||||
it("读取正整数,非法值回退默认", () => {
|
||||
process.env.TEST_INT_ENV = "42";
|
||||
expect(intFromEnv("TEST_INT_ENV", 7)).toBe(42);
|
||||
|
||||
process.env.TEST_INT_ENV = "abc";
|
||||
expect(intFromEnv("TEST_INT_ENV", 7)).toBe(7);
|
||||
|
||||
process.env.TEST_INT_ENV = "-1";
|
||||
expect(intFromEnv("TEST_INT_ENV", 7)).toBe(7);
|
||||
|
||||
delete process.env.TEST_INT_ENV;
|
||||
expect(intFromEnv("TEST_INT_ENV", 7)).toBe(7);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,68 @@
|
||||
/**
|
||||
* 进程内限流(安全-03)。
|
||||
*
|
||||
* 目的:`/ai/tasks` 每次调用都会消耗 LLM 额度,需要防止误用刷爆。
|
||||
* 不引入额外依赖,按「每窗口每来源」计数即可。
|
||||
*
|
||||
* 注意:这是单进程内存计数。多副本部署时每个副本各算一份,属于粗粒度保护,
|
||||
* 不是精确配额;真要做精确配额需要放到共享存储(当前不需要)。
|
||||
*/
|
||||
|
||||
export interface RateLimitOptions {
|
||||
/** 窗口内允许的请求数 */
|
||||
limit: number;
|
||||
/** 窗口长度(毫秒) */
|
||||
windowMs: number;
|
||||
}
|
||||
|
||||
export interface RateLimitDecision {
|
||||
allowed: boolean;
|
||||
/** 建议告诉调用方还要等多久(毫秒) */
|
||||
retryAfterMs: number;
|
||||
}
|
||||
|
||||
export class RateLimiter {
|
||||
private readonly hits = new Map<string, number[]>();
|
||||
private lastSweep = 0;
|
||||
|
||||
constructor(private readonly options: RateLimitOptions) {}
|
||||
|
||||
/** 记一次命中并返回是否放行 */
|
||||
check(key: string, now = Date.now()): RateLimitDecision {
|
||||
this.sweep(now);
|
||||
|
||||
const windowStart = now - this.options.windowMs;
|
||||
const recent = (this.hits.get(key) ?? []).filter((at) => at > windowStart);
|
||||
if (recent.length >= this.options.limit) {
|
||||
this.hits.set(key, recent);
|
||||
const oldest = recent[0];
|
||||
return { allowed: false, retryAfterMs: Math.max(0, oldest + this.options.windowMs - now) };
|
||||
}
|
||||
recent.push(now);
|
||||
this.hits.set(key, recent);
|
||||
return { allowed: true, retryAfterMs: 0 };
|
||||
}
|
||||
|
||||
/** 清理过期的 key,避免长期运行后 Map 无界增长 */
|
||||
private sweep(now: number): void {
|
||||
if (now - this.lastSweep < this.options.windowMs) return;
|
||||
this.lastSweep = now;
|
||||
const windowStart = now - this.options.windowMs;
|
||||
for (const [key, times] of this.hits) {
|
||||
const recent = times.filter((at) => at > windowStart);
|
||||
if (recent.length === 0) {
|
||||
this.hits.delete(key);
|
||||
} else {
|
||||
this.hits.set(key, recent);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 从环境变量读一个正整数,非法时返回默认值 */
|
||||
export function intFromEnv(name: string, fallback: number): number {
|
||||
const raw = process.env[name];
|
||||
if (!raw) return fallback;
|
||||
const value = Number(raw);
|
||||
return Number.isInteger(value) && value > 0 ? value : fallback;
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { isBlockedAddress, UnsafeUrlError, assertSafeUrl } from "./ssrf-guard.js";
|
||||
|
||||
describe("isBlockedAddress — IPv4", () => {
|
||||
it("拦下私网、回环、链路本地(含云元数据)与保留段", () => {
|
||||
const blocked = [
|
||||
"127.0.0.1",
|
||||
"127.255.255.254",
|
||||
"10.0.0.1",
|
||||
"10.255.255.255",
|
||||
"172.16.0.1",
|
||||
"172.31.255.255",
|
||||
"192.168.1.1",
|
||||
"169.254.169.254", // 云元数据
|
||||
"0.0.0.0",
|
||||
"100.64.0.1", // CGNAT
|
||||
"224.0.0.1", // 组播
|
||||
"255.255.255.255",
|
||||
];
|
||||
for (const ip of blocked) {
|
||||
expect(isBlockedAddress(ip), ip).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("放行公网地址", () => {
|
||||
const allowed = ["8.8.8.8", "1.1.1.1", "100.63.255.255", "172.32.0.1", "192.169.0.1", "223.255.255.255"];
|
||||
for (const ip of allowed) {
|
||||
expect(isBlockedAddress(ip), ip).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it("非法 IPv4 字面量按不安全处理", () => {
|
||||
expect(isBlockedAddress("999.1.1.1")).toBe(true);
|
||||
expect(isBlockedAddress("1.2.3")).toBe(true);
|
||||
expect(isBlockedAddress("")).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isBlockedAddress — IPv6", () => {
|
||||
it("拦下回环、未指定、ULA、链路本地与组播", () => {
|
||||
const blocked = ["::1", "::", "fc00::1", "fd12:3456::1", "fe80::1", "ff02::1", "fe80::1%eth0"];
|
||||
for (const ip of blocked) {
|
||||
expect(isBlockedAddress(ip), ip).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("识别 IPv4-mapped 内嵌的私网地址(绕过尝试)", () => {
|
||||
expect(isBlockedAddress("::ffff:127.0.0.1")).toBe(true);
|
||||
expect(isBlockedAddress("::ffff:169.254.169.254")).toBe(true);
|
||||
expect(isBlockedAddress("::ffff:8.8.8.8")).toBe(false);
|
||||
});
|
||||
|
||||
it("识别 NAT64 内嵌的私网地址", () => {
|
||||
expect(isBlockedAddress("64:ff9b::7f00:1")).toBe(true); // 内嵌 127.0.0.1
|
||||
expect(isBlockedAddress("64:ff9b::808:808")).toBe(false); // 内嵌 8.8.8.8
|
||||
});
|
||||
|
||||
it("放行公网 IPv6", () => {
|
||||
expect(isBlockedAddress("2001:4860:4860::8888")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("assertSafeUrl", () => {
|
||||
it("只允许 http/https", async () => {
|
||||
await expect(assertSafeUrl("file:///etc/passwd")).rejects.toBeInstanceOf(UnsafeUrlError);
|
||||
await expect(assertSafeUrl("ftp://example.com")).rejects.toBeInstanceOf(UnsafeUrlError);
|
||||
await expect(assertSafeUrl("gopher://example.com")).rejects.toBeInstanceOf(UnsafeUrlError);
|
||||
});
|
||||
|
||||
it("拒绝无效 URL", async () => {
|
||||
await expect(assertSafeUrl("not a url")).rejects.toBeInstanceOf(UnsafeUrlError);
|
||||
});
|
||||
|
||||
it("拒绝内网 IP 字面量,不经过 DNS", async () => {
|
||||
await expect(assertSafeUrl("http://127.0.0.1:80/")).rejects.toBeInstanceOf(UnsafeUrlError);
|
||||
await expect(assertSafeUrl("http://192.168.1.1/")).rejects.toBeInstanceOf(UnsafeUrlError);
|
||||
await expect(assertSafeUrl("http://169.254.169.254/latest/meta-data/")).rejects.toBeInstanceOf(
|
||||
UnsafeUrlError,
|
||||
);
|
||||
await expect(assertSafeUrl("http://[::1]/")).rejects.toBeInstanceOf(UnsafeUrlError);
|
||||
});
|
||||
|
||||
it("拒绝非 http/https 默认端口的地址(避免探测内网服务)", async () => {
|
||||
await expect(assertSafeUrl("http://8.8.8.8:6379/")).rejects.toBeInstanceOf(UnsafeUrlError);
|
||||
await expect(assertSafeUrl("http://8.8.8.8:3306/")).rejects.toBeInstanceOf(UnsafeUrlError);
|
||||
});
|
||||
|
||||
it("放行公网 http/https 字面量与默认端口", async () => {
|
||||
await expect(assertSafeUrl("http://8.8.8.8/")).resolves.toBeInstanceOf(URL);
|
||||
await expect(assertSafeUrl("https://8.8.8.8:443/")).resolves.toBeInstanceOf(URL);
|
||||
});
|
||||
|
||||
it("域名解析失败时拒绝", async () => {
|
||||
await expect(assertSafeUrl("https://this-domain-should-not-resolve.invalid/")).rejects.toBeInstanceOf(
|
||||
UnsafeUrlError,
|
||||
);
|
||||
});
|
||||
|
||||
it("localhost 这类解析到回环的域名被拦下(防绕过字面量检查)", async () => {
|
||||
await expect(assertSafeUrl("http://localhost:80/")).rejects.toBeInstanceOf(UnsafeUrlError);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,285 @@
|
||||
/**
|
||||
* SSRF 防护:校验「服务端主动抓取」的目标地址是否安全。
|
||||
*
|
||||
* 背景:`/fetch-title` 由服务端抓取调用方给的 URL。如果不做限制,调用方可以拿它探测
|
||||
* 内网服务与云元数据地址(169.254.169.254),把本服务当成内网跳板。
|
||||
*
|
||||
* 防护分四层,缺一层都能绕过:
|
||||
* 1. 协议只允许 http/https;
|
||||
* 2. 端口只允许 http/https 默认端口(外加显式配置的额外端口),避免被用来探测 Redis/MySQL 等内网服务;
|
||||
* 3. 解析域名后校验**每一个**解析结果,而不是只校验字面量(防 DNS 解析到内网、防 DNS rebinding);
|
||||
* 4. 重定向逐跳校验,避免「首跳合法、次跳内网」。
|
||||
*/
|
||||
|
||||
import { lookup } from "node:dns/promises";
|
||||
|
||||
/** 允许的协议 */
|
||||
const ALLOWED_PROTOCOLS = new Set(["http:", "https:"]);
|
||||
|
||||
/** 默认允许的端口 */
|
||||
const DEFAULT_ALLOWED_PORTS = new Set(["80", "443"]);
|
||||
|
||||
/** 响应体读取上限(字节):标题通常在头部,不必读完整个页面 */
|
||||
export const MAX_RESPONSE_BYTES = 512 * 1024;
|
||||
|
||||
/** 最大重定向跳数 */
|
||||
export const MAX_REDIRECTS = 3;
|
||||
|
||||
export class UnsafeUrlError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "UnsafeUrlError";
|
||||
}
|
||||
}
|
||||
|
||||
/** 解析额外允许的端口:环境变量 LPT_AI_FETCH_ALLOWED_PORTS,逗号分隔 */
|
||||
function allowedPorts(): Set<string> {
|
||||
const extra = (process.env.LPT_AI_FETCH_ALLOWED_PORTS ?? "")
|
||||
.split(",")
|
||||
.map((p) => p.trim())
|
||||
.filter((p) => /^\d+$/.test(p));
|
||||
return new Set([...DEFAULT_ALLOWED_PORTS, ...extra]);
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断一个 IP 字面量是否属于「不可主动访问」的地址段。
|
||||
*
|
||||
* 覆盖 IPv4 私网/回环/链路本地(含云元数据 169.254.169.254)/CGNAT/组播/保留段,
|
||||
* 以及 IPv6 回环、未指定、ULA、链路本地、组播,并识别 IPv4-mapped 与 NAT64 里内嵌的 IPv4。
|
||||
*/
|
||||
export function isBlockedAddress(address: string): boolean {
|
||||
const ip = address.trim().toLowerCase();
|
||||
if (!ip) return true;
|
||||
|
||||
// IPv6:去掉可能的 zone id(fe80::1%eth0)
|
||||
const bare = ip.includes("%") ? ip.slice(0, ip.indexOf("%")) : ip;
|
||||
|
||||
if (bare.includes(":")) {
|
||||
return isBlockedIpv6(bare);
|
||||
}
|
||||
return isBlockedIpv4(bare);
|
||||
}
|
||||
|
||||
function isBlockedIpv4(ip: string): boolean {
|
||||
const parts = ip.split(".");
|
||||
if (parts.length !== 4) return true; // 不是合法 IPv4,按不安全处理
|
||||
const octets = parts.map((p) => Number(p));
|
||||
if (octets.some((n) => !Number.isInteger(n) || n < 0 || n > 255)) return true;
|
||||
|
||||
const [a, b] = octets;
|
||||
if (a === 0) return true; // 0.0.0.0/8
|
||||
if (a === 10) return true; // 10.0.0.0/8
|
||||
if (a === 127) return true; // 回环
|
||||
if (a === 169 && b === 254) return true; // 链路本地,含云元数据
|
||||
if (a === 172 && b >= 16 && b <= 31) return true; // 172.16.0.0/12
|
||||
if (a === 192 && b === 168) return true; // 192.168.0.0/16
|
||||
if (a === 192 && b === 0) return true; // 192.0.0.0/24 与 192.0.2.0/24
|
||||
if (a === 100 && b >= 64 && b <= 127) return true; // CGNAT 100.64.0.0/10
|
||||
if (a === 198 && (b === 18 || b === 19)) return true; // 基准测试 198.18.0.0/15
|
||||
if (a === 198 && b === 51) return true; // TEST-NET-2
|
||||
if (a === 203 && b === 0) return true; // TEST-NET-3
|
||||
if (a >= 224) return true; // 组播与保留段
|
||||
return false;
|
||||
}
|
||||
|
||||
function isBlockedIpv6(ip: string): boolean {
|
||||
// IPv4-mapped:::ffff:127.0.0.1 / ::ffff:7f00:1
|
||||
const mapped = ip.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/);
|
||||
if (mapped) return isBlockedIpv4(mapped[1]);
|
||||
|
||||
const groups = expandIpv6(ip);
|
||||
if (!groups) return true; // 无法解析,按不安全处理
|
||||
|
||||
const first = groups[0];
|
||||
if (groups.every((g) => g === 0)) return true; // :: 未指定
|
||||
if (groups.slice(0, 7).every((g) => g === 0) && groups[7] === 1) return true; // ::1 回环
|
||||
if ((first & 0xfe00) === 0xfc00) return true; // fc00::/7 ULA
|
||||
if ((first & 0xffc0) === 0xfe80) return true; // fe80::/10 链路本地
|
||||
if ((first & 0xff00) === 0xff00) return true; // ff00::/8 组播
|
||||
|
||||
// NAT64 64:ff9b::/96:内嵌 IPv4,取最后两组
|
||||
if (groups[0] === 0x0064 && groups[1] === 0xff9b) {
|
||||
const embedded = [groups[6] >> 8, groups[6] & 0xff, groups[7] >> 8, groups[7] & 0xff].join(".");
|
||||
return isBlockedIpv4(embedded);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** 把 IPv6 文本展开成 8 组 16 位整数;无法解析返回 null */
|
||||
function expandIpv6(ip: string): number[] | null {
|
||||
let text = ip;
|
||||
// 内嵌 IPv4(如 ::ffff:1.2.3.4)先转成两组十六进制
|
||||
const embedded = text.match(/^(.*:)(\d+\.\d+\.\d+\.\d+)$/);
|
||||
if (embedded) {
|
||||
const octets = embedded[2].split(".").map((p) => Number(p));
|
||||
if (octets.some((n) => !Number.isInteger(n) || n < 0 || n > 255)) return null;
|
||||
const high = ((octets[0] << 8) | octets[1]).toString(16);
|
||||
const low = ((octets[2] << 8) | octets[3]).toString(16);
|
||||
text = `${embedded[1]}${high}:${low}`;
|
||||
}
|
||||
|
||||
const halves = text.split("::");
|
||||
if (halves.length > 2) return null;
|
||||
|
||||
const parseGroups = (part: string): number[] | null => {
|
||||
if (!part) return [];
|
||||
const items = part.split(":");
|
||||
const result: number[] = [];
|
||||
for (const item of items) {
|
||||
if (!/^[0-9a-f]{1,4}$/.test(item)) return null;
|
||||
result.push(parseInt(item, 16));
|
||||
}
|
||||
return result;
|
||||
};
|
||||
|
||||
const head = parseGroups(halves[0]);
|
||||
const tail = halves.length === 2 ? parseGroups(halves[1]) : [];
|
||||
if (head === null || tail === null) return null;
|
||||
|
||||
if (halves.length === 1) {
|
||||
return head.length === 8 ? head : null;
|
||||
}
|
||||
const fill = 8 - head.length - tail.length;
|
||||
if (fill < 0) return null;
|
||||
return [...head, ...new Array(fill).fill(0), ...tail];
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验目标 URL 是否可以安全抓取。不通过时抛 {@link UnsafeUrlError}。
|
||||
*
|
||||
* @param rawUrl 调用方给的 URL
|
||||
*/
|
||||
export async function assertSafeUrl(rawUrl: string): Promise<URL> {
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(rawUrl);
|
||||
} catch {
|
||||
throw new UnsafeUrlError("无效的 URL");
|
||||
}
|
||||
|
||||
if (!ALLOWED_PROTOCOLS.has(url.protocol)) {
|
||||
throw new UnsafeUrlError("只支持抓取 http/https 链接");
|
||||
}
|
||||
|
||||
const port = url.port || (url.protocol === "https:" ? "443" : "80");
|
||||
if (!allowedPorts().has(port)) {
|
||||
throw new UnsafeUrlError("这个链接的端口不在允许范围内");
|
||||
}
|
||||
|
||||
// 字面量就是 IP 时直接判定,避免走了 DNS 分支
|
||||
const host = url.hostname.replace(/^\[|\]$/g, "");
|
||||
if (isIpLiteral(host)) {
|
||||
if (isBlockedAddress(host)) {
|
||||
throw new UnsafeUrlError("不能抓取内网或本机地址");
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
// 域名:解析后逐个校验,防止解析到内网或 DNS rebinding
|
||||
let addresses: string[];
|
||||
try {
|
||||
const resolved = await lookup(host, { all: true });
|
||||
addresses = resolved.map((item) => item.address);
|
||||
} catch {
|
||||
throw new UnsafeUrlError("无法解析这个链接的域名");
|
||||
}
|
||||
if (addresses.length === 0) {
|
||||
throw new UnsafeUrlError("无法解析这个链接的域名");
|
||||
}
|
||||
if (addresses.some((address) => isBlockedAddress(address))) {
|
||||
throw new UnsafeUrlError("不能抓取内网或本机地址");
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
function isIpLiteral(host: string): boolean {
|
||||
if (/^\d+\.\d+\.\d+\.\d+$/.test(host)) return true;
|
||||
return host.includes(":");
|
||||
}
|
||||
|
||||
export interface SafeFetchResult {
|
||||
/** 最终跳转到的地址 */
|
||||
finalUrl: string;
|
||||
/** 响应体(截断到上限) */
|
||||
body: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* 带上限与逐跳校验的抓取。
|
||||
*
|
||||
* @param rawUrl 目标地址
|
||||
* @param timeoutMs 单次请求超时
|
||||
*/
|
||||
export async function safeFetchText(rawUrl: string, timeoutMs = 10_000): Promise<SafeFetchResult> {
|
||||
let current = await assertSafeUrl(rawUrl);
|
||||
|
||||
for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
const response = await fetch(current, {
|
||||
signal: controller.signal,
|
||||
// 自己处理重定向,才能逐跳校验
|
||||
redirect: "manual",
|
||||
headers: { accept: "text/html,application/xhtml+xml" },
|
||||
});
|
||||
|
||||
if (isRedirect(response.status)) {
|
||||
const location = response.headers.get("location");
|
||||
if (!location) {
|
||||
throw new UnsafeUrlError("重定向没有给出目标地址");
|
||||
}
|
||||
// 相对跳转按当前地址解析,然后重新走一遍完整校验
|
||||
current = await assertSafeUrl(new URL(location, current).toString());
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
throw new UnsafeUrlError("目标地址没有正常返回内容");
|
||||
}
|
||||
|
||||
const body = await readCapped(response);
|
||||
return { finalUrl: current.toString(), body };
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
throw new UnsafeUrlError("跳转次数过多,已停止抓取");
|
||||
}
|
||||
|
||||
function isRedirect(status: number): boolean {
|
||||
return status === 301 || status === 302 || status === 303 || status === 307 || status === 308;
|
||||
}
|
||||
|
||||
/**
|
||||
* 读取响应体并截断:既防超大响应打爆内存,也避免为一个标题读完整个页面。
|
||||
*/
|
||||
async function readCapped(response: Response): Promise<string> {
|
||||
const contentType = response.headers.get("content-type") ?? "";
|
||||
if (contentType && !/text\/|xml|json/i.test(contentType)) {
|
||||
throw new UnsafeUrlError("这个链接不是网页内容");
|
||||
}
|
||||
|
||||
const body = response.body;
|
||||
if (!body) return "";
|
||||
|
||||
const reader = body.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let total = 0;
|
||||
try {
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
if (!value) continue;
|
||||
chunks.push(value);
|
||||
total += value.byteLength;
|
||||
if (total >= MAX_RESPONSE_BYTES) break;
|
||||
}
|
||||
} finally {
|
||||
await reader.cancel().catch(() => undefined);
|
||||
}
|
||||
|
||||
const buffer = Buffer.concat(chunks.map((chunk) => Buffer.from(chunk)));
|
||||
const text = buffer.subarray(0, MAX_RESPONSE_BYTES).toString("utf-8");
|
||||
return text;
|
||||
}
|
||||
Reference in New Issue
Block a user