feat: 加固标题抓取并加上限流与可选入站令牌
This commit is contained in:
+53
-11
@@ -6,10 +6,37 @@ import type { FastifyInstance } from "fastify";
|
||||
import { isAvailable, loadConfig } from "../llm/client.js";
|
||||
import { createTask, getTask, type TaskType } from "../task-queue.js";
|
||||
import { logStore } from "../admin/store.js";
|
||||
import { safeFetchText, UnsafeUrlError } from "../utils/ssrf-guard.js";
|
||||
import { RateLimiter, intFromEnv } from "../utils/rate-limit.js";
|
||||
import { checkToken } from "../utils/auth.js";
|
||||
|
||||
export async function aiRoutes(app: FastifyInstance): Promise<void> {
|
||||
const config = loadConfig();
|
||||
|
||||
// 安全-03:AI 任务会消耗 LLM 额度,按来源限流(单进程粗粒度保护)
|
||||
const taskLimiter = new RateLimiter({
|
||||
limit: intFromEnv("LPT_AI_RATE_LIMIT", 30),
|
||||
windowMs: intFromEnv("LPT_AI_RATE_WINDOW_MS", 60_000),
|
||||
});
|
||||
|
||||
// 安全-03:可选的入站令牌。设置了 LPT_AI_TOKEN 时,所有 /ai/* 请求都必须携带
|
||||
app.addHook("preHandler", async (request, reply) => {
|
||||
if (!request.url.startsWith("/ai/")) return;
|
||||
if (!checkToken(request, reply)) {
|
||||
return reply;
|
||||
}
|
||||
return undefined;
|
||||
});
|
||||
|
||||
/** 取限流用的来源标识:优先真实客户端 IP(后面有反代时取 X-Forwarded-For 首段) */
|
||||
const clientKey = (request: { ip?: string; headers: Record<string, unknown> }): string => {
|
||||
const forwarded = request.headers["x-forwarded-for"];
|
||||
if (typeof forwarded === "string" && forwarded.trim()) {
|
||||
return forwarded.split(",")[0].trim();
|
||||
}
|
||||
return request.ip ?? "unknown";
|
||||
};
|
||||
|
||||
app.get("/health", async () => ({
|
||||
status: "ok",
|
||||
llmAvailable: isAvailable(config),
|
||||
@@ -29,6 +56,15 @@ export async function aiRoutes(app: FastifyInstance): Promise<void> {
|
||||
}>("/ai/tasks", async (request, reply) => {
|
||||
const { type, params, idempotencyKey, force } = request.body ?? {};
|
||||
|
||||
const decision = taskLimiter.check(clientKey(request));
|
||||
if (!decision.allowed) {
|
||||
app.log.warn({ key: clientKey(request) }, "AI 任务提交被限流");
|
||||
return reply
|
||||
.status(429)
|
||||
.header("retry-after", String(Math.ceil(decision.retryAfterMs / 1000)))
|
||||
.send({ error: "提交太频繁了,请稍后再试" });
|
||||
}
|
||||
|
||||
if (!type || !["aggregate-report", "generate-mind-map", "compare-recall"].includes(type)) {
|
||||
return reply.status(400).send({ error: "无效的 type,允许: aggregate-report, generate-mind-map, compare-recall" });
|
||||
}
|
||||
@@ -58,23 +94,23 @@ export async function aiRoutes(app: FastifyInstance): Promise<void> {
|
||||
return task;
|
||||
});
|
||||
|
||||
/** 抓取网页标题 */
|
||||
/**
|
||||
* 抓取网页标题。
|
||||
*
|
||||
* <p>安全-03:目标由调用方给出,因此在抓取前做协议、端口、DNS 解析与逐跳重定向校验,
|
||||
* 避免被当成探测内网与云元数据的跳板,同时对响应体做截断。</p>
|
||||
*/
|
||||
app.post<{ Body: { url: string } }>("/fetch-title", async (request, reply) => {
|
||||
const { url } = request.body ?? {};
|
||||
const t0 = Date.now();
|
||||
|
||||
if (!url || typeof url !== "string" || (!url.startsWith("http://") && !url.startsWith("https://"))) {
|
||||
if (!url || typeof url !== "string") {
|
||||
return reply.status(400).send({ error: "无效的 URL" });
|
||||
}
|
||||
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), 10_000);
|
||||
const response = await fetch(url, { signal: controller.signal });
|
||||
clearTimeout(timer);
|
||||
|
||||
const html = await response.text();
|
||||
const match = html.match(/<title[^>]*>([^<]+)<\/title>/i);
|
||||
const { body } = await safeFetchText(url);
|
||||
const match = body.match(/<title[^>]*>([^<]+)<\/title>/i);
|
||||
const title = match ? match[1].trim() : null;
|
||||
|
||||
logStore.add({
|
||||
@@ -88,16 +124,22 @@ export async function aiRoutes(app: FastifyInstance): Promise<void> {
|
||||
});
|
||||
return { url, title };
|
||||
} catch (err) {
|
||||
// 被安全校验拦下时返回 400(调用方的问题),其余按抓取失败处理
|
||||
const blocked = err instanceof UnsafeUrlError;
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
logStore.add({
|
||||
timestamp: new Date().toISOString(),
|
||||
endpoint: "/fetch-title",
|
||||
model: "",
|
||||
durationMs: Date.now() - t0,
|
||||
status: 502,
|
||||
status: blocked ? 400 : 502,
|
||||
requestBody: { url },
|
||||
responseBody: { url, title: null },
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
error: message,
|
||||
});
|
||||
if (blocked) {
|
||||
return reply.status(400).send({ error: message });
|
||||
}
|
||||
return { url, title: null };
|
||||
}
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user