1052 lines
35 KiB
JavaScript
1052 lines
35 KiB
JavaScript
import assert from "node:assert/strict";
|
||
import { spawn } from "node:child_process";
|
||
import { mkdirSync, rmSync } from "node:fs";
|
||
import { DatabaseSync } from "node:sqlite";
|
||
import path from "node:path";
|
||
import { fileURLToPath } from "node:url";
|
||
import {
|
||
classifyContinuationReply,
|
||
detectConversationContinuation,
|
||
isActionableSkillContinuation,
|
||
isLikelySkillContinuationReply,
|
||
} from "../src/lib/skill-continuation.ts";
|
||
import {
|
||
findUnsupportedSkillCapabilities,
|
||
getSkillCapabilityViolations,
|
||
} from "../src/lib/skill-capabilities.ts";
|
||
import { isSuggestionWithinNodeScope } from "../src/lib/skill-recommendation-scope.ts";
|
||
import {
|
||
evaluateSkillNodeQuality,
|
||
getSkillNodeQualityIssues,
|
||
} from "../src/lib/skill-node-quality.ts";
|
||
import { estimateDeepSeekCallUsage } from "../src/lib/model-usage.ts";
|
||
|
||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||
const port = 3310 + (process.pid % 200);
|
||
const baseUrl = `http://127.0.0.1:${port}`;
|
||
const tempDir = path.join(root, ".acceptance");
|
||
const databasePath = path.join(tempDir, `skillloom-${process.pid}.db`);
|
||
const nextBin = path.join(root, "node_modules", "next", "dist", "bin", "next");
|
||
|
||
mkdirSync(tempDir, { recursive: true });
|
||
const legacyDatabase = new DatabaseSync(databasePath);
|
||
legacyDatabase.exec(`
|
||
CREATE TABLE conversations (
|
||
id TEXT PRIMARY KEY,
|
||
title TEXT NOT NULL,
|
||
retained_skill_ids TEXT NOT NULL DEFAULT '[]',
|
||
created_at TEXT NOT NULL,
|
||
updated_at TEXT NOT NULL
|
||
);
|
||
INSERT INTO conversations
|
||
(id, title, retained_skill_ids, created_at, updated_at)
|
||
VALUES
|
||
('legacy_conversation', '', '["skill_sql_guard"]',
|
||
'2026-01-01T00:00:00.000Z', '2026-01-01T00:00:00.000Z');
|
||
`);
|
||
legacyDatabase.close();
|
||
|
||
const server = spawn(process.execPath, [nextBin, "start", "-p", String(port)], {
|
||
cwd: root,
|
||
env: {
|
||
...process.env,
|
||
DATABASE_PATH: databasePath,
|
||
DEEPSEEK_API_KEY: "",
|
||
NEXT_TELEMETRY_DISABLED: "1",
|
||
},
|
||
stdio: ["ignore", "pipe", "pipe"],
|
||
});
|
||
|
||
let serverLog = "";
|
||
server.stdout.on("data", (chunk) => {
|
||
serverLog += chunk.toString();
|
||
});
|
||
server.stderr.on("data", (chunk) => {
|
||
serverLog += chunk.toString();
|
||
});
|
||
|
||
async function waitForServer() {
|
||
for (let attempt = 0; attempt < 60; attempt += 1) {
|
||
try {
|
||
const response = await fetch(`${baseUrl}/api/chat`);
|
||
if (response.ok) return;
|
||
} catch {
|
||
// The server has not opened its port yet.
|
||
}
|
||
await new Promise((resolve) => setTimeout(resolve, 250));
|
||
}
|
||
throw new Error(`测试服务未启动：\n${serverLog}`);
|
||
}
|
||
|
||
async function jsonRequest(url, init) {
|
||
const response = await fetch(`${baseUrl}${url}`, init);
|
||
const payload = await response.json().catch(() => null);
|
||
if (!response.ok) {
|
||
throw new Error(`${init?.method ?? "GET"} ${url}: ${JSON.stringify(payload)}`);
|
||
}
|
||
return payload;
|
||
}
|
||
|
||
async function readSse(response) {
|
||
assert.equal(response.ok, true, `SSE 请求失败：${response.status}`);
|
||
assert.ok(response.body, "SSE 响应没有 body");
|
||
const reader = response.body.getReader();
|
||
const decoder = new TextDecoder();
|
||
let buffer = "";
|
||
const events = [];
|
||
|
||
while (true) {
|
||
const { done, value } = await reader.read();
|
||
if (done) break;
|
||
buffer += decoder.decode(value, { stream: true });
|
||
const blocks = buffer.split(/\r?\n\r?\n/);
|
||
buffer = blocks.pop() ?? "";
|
||
for (const block of blocks) {
|
||
const data = block
|
||
.split(/\r?\n/)
|
||
.filter((line) => line.startsWith("data:"))
|
||
.map((line) => line.slice(5).trim())
|
||
.join("\n");
|
||
if (data) events.push(JSON.parse(data));
|
||
}
|
||
}
|
||
return events;
|
||
}
|
||
|
||
async function readFirstSseBatch(response) {
|
||
assert.ok(response.body, "SSE 响应没有 body");
|
||
const reader = response.body.getReader();
|
||
const decoder = new TextDecoder();
|
||
const { value } = await reader.read();
|
||
await reader.cancel();
|
||
const text = decoder.decode(value);
|
||
return text
|
||
.split(/\r?\n\r?\n/)
|
||
.map((block) =>
|
||
block
|
||
.split(/\r?\n/)
|
||
.filter((line) => line.startsWith("data:"))
|
||
.map((line) => line.slice(5).trim())
|
||
.join("\n"),
|
||
)
|
||
.filter(Boolean)
|
||
.map((data) => JSON.parse(data));
|
||
}
|
||
|
||
const definitions = [
|
||
["trigger", "触发条件", "什么时候应该使用这个 Skill，以及不适用的边界。"],
|
||
["inputs", "输入参数", "执行所需的信息、字段、格式和默认值。"],
|
||
["steps", "执行步骤", "AI 应严格遵循的行动顺序与关键判断。"],
|
||
["output", "输出格式", "最终交付物的结构、字段、语气和示例。"],
|
||
["constraints", "约束与测试", "禁止事项、质量门槛、失败处理和验收用例。"],
|
||
];
|
||
|
||
let nodes = definitions.map(([key, title, description]) => ({
|
||
key,
|
||
title,
|
||
description,
|
||
content: "",
|
||
ready: false,
|
||
completed: false,
|
||
}));
|
||
let skillName = "未命名 Skill";
|
||
let skillDescription = "";
|
||
let builderMessages = [];
|
||
|
||
async function builderTurn(message) {
|
||
const clientRequestId = `acceptance_builder_${crypto.randomUUID()}`;
|
||
const requestBody = (afterSeq) =>
|
||
JSON.stringify({
|
||
clientRequestId,
|
||
afterSeq,
|
||
skillId: null,
|
||
message,
|
||
skillName,
|
||
skillDescription,
|
||
nodes,
|
||
messages: builderMessages,
|
||
});
|
||
const response = await fetch(`${baseUrl}/api/skills/builder`, {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: requestBody(0),
|
||
});
|
||
|
||
let events;
|
||
if (builderMessages.length === 0) {
|
||
const firstEvents = await readFirstSseBatch(response);
|
||
const cursor = Math.max(...firstEvents.map((event) => event.seq ?? 0));
|
||
const resumed = await fetch(`${baseUrl}/api/skills/builder`, {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: requestBody(cursor),
|
||
});
|
||
events = [...firstEvents, ...(await readSse(resumed))];
|
||
const sequences = events.map((event) => event.seq).filter(Boolean);
|
||
assert.equal(
|
||
new Set(sequences).size,
|
||
sequences.length,
|
||
"Builder 重连不得重复事件",
|
||
);
|
||
} else {
|
||
events = await readSse(response);
|
||
}
|
||
const update = events.find((event) => event.type === "builder_update");
|
||
assert.ok(update, "Builder 必须返回结构化节点更新");
|
||
assert.ok(
|
||
Array.isArray(update.evaluation.suggestions) &&
|
||
update.evaluation.suggestions.length >= 2 &&
|
||
update.evaluation.suggestions.length <= 4,
|
||
"Builder 必须返回 2-4 条结构化 AI 智能建议",
|
||
);
|
||
assert.equal(
|
||
update.evaluation.suggestionNodeKey,
|
||
update.evaluation.activeNode,
|
||
"Builder 推荐必须明确归属于当前待完善节点",
|
||
);
|
||
assert.equal(
|
||
update.evaluation.nodeQuality.length,
|
||
5,
|
||
"Builder 必须返回五节点质量判定，便于排查进度原因",
|
||
);
|
||
assert.ok(
|
||
events.some((event) => event.type === "complete"),
|
||
"Builder 必须发出完成事件",
|
||
);
|
||
nodes = update.evaluation.nodes;
|
||
skillName = update.evaluation.skillName;
|
||
skillDescription = update.evaluation.skillDescription;
|
||
const reply = events
|
||
.filter((event) => event.type === "token")
|
||
.map((event) => event.token)
|
||
.join("");
|
||
builderMessages.push(
|
||
{ role: "user", content: message },
|
||
{ role: "assistant", content: reply },
|
||
);
|
||
return { evaluation: update.evaluation, reply };
|
||
}
|
||
|
||
async function runAcceptance() {
|
||
await waitForServer();
|
||
console.log("✓ 测试服务与独立 SQLite 已启动");
|
||
|
||
const migratedChat = await jsonRequest("/api/chat");
|
||
assert.equal(migratedChat.continuation.state, "awaiting_input");
|
||
assert.deepEqual(migratedChat.continuation.skillIds, ["skill_sql_guard"]);
|
||
assert.equal(migratedChat.continuation.source, "fallback");
|
||
console.log("✓ 旧版 retained_skill_ids 已兼容迁移为显式续接状态");
|
||
|
||
const pricedUsage = estimateDeepSeekCallUsage({
|
||
model: "deepseek-v4-flash",
|
||
promptTokens: 3_000,
|
||
promptCacheHitTokens: 1_000,
|
||
promptCacheMissTokens: 2_000,
|
||
completionTokens: 800,
|
||
});
|
||
assert.equal(pricedUsage.totalTokens, 3_800);
|
||
assert.equal(
|
||
pricedUsage.estimatedCostMicros,
|
||
3_620,
|
||
"费用必须分别按缓存命中输入、未命中输入和输出计算",
|
||
);
|
||
console.log("✓ Token 用量按模型价格快照计算预计费用");
|
||
|
||
const concreteOffer =
|
||
"需要我根据你的实际可用时间调整这份学习计划吗？";
|
||
assert.equal(
|
||
isActionableSkillContinuation(concreteOffer),
|
||
true,
|
||
"带有明确动作和对象的后续提议必须保留 Skill",
|
||
);
|
||
assert.equal(
|
||
detectConversationContinuation(concreteOffer)?.state,
|
||
"offer_pending",
|
||
"具体后续提议必须显式记录为等待确认",
|
||
);
|
||
assert.equal(classifyContinuationReply("需要"), "accept");
|
||
assert.equal(classifyContinuationReply("每天 1 小时"), "answer");
|
||
assert.equal(classifyContinuationReply("不用了"), "decline");
|
||
assert.equal(
|
||
isActionableSkillContinuation("如果需要，我可以继续帮助你。"),
|
||
false,
|
||
"泛泛的礼貌性邀请不得让 Skill 一直保留",
|
||
);
|
||
assert.equal(
|
||
isLikelySkillContinuationReply("需要", [
|
||
{ role: "assistant", content: concreteOffer },
|
||
]),
|
||
true,
|
||
"简短的接受回复必须识别为上一轮的延续",
|
||
);
|
||
assert.equal(
|
||
isLikelySkillContinuationReply("帮我查一下天气", [
|
||
{ role: "assistant", content: concreteOffer },
|
||
]),
|
||
false,
|
||
"用户切换到新任务时不得强制沿用旧 Skill",
|
||
);
|
||
console.log("✓ 具体后续提议可续接，礼貌邀请与新话题不会粘住 Skill");
|
||
|
||
assert.deepEqual(
|
||
findUnsupportedSkillCapabilities(
|
||
"自动调用外部 API 获取数据，再运行 Python 脚本生成报告。",
|
||
),
|
||
["运行脚本、代码或命令", "调用外部 API 或第三方服务"],
|
||
"必须识别当前工作台无法执行的能力承诺",
|
||
);
|
||
assert.deepEqual(
|
||
findUnsupportedSkillCapabilities(
|
||
"审阅用户提供的 API 文档；不得调用外部 API 或运行本地脚本。",
|
||
),
|
||
[],
|
||
"允许审阅 API/脚本内容，也允许把禁止调用写入约束",
|
||
);
|
||
assert.deepEqual(
|
||
findUnsupportedSkillCapabilities(
|
||
"指导用户如何调用 API，并要求用户手动运行脚本后粘贴结果。",
|
||
),
|
||
[],
|
||
"允许提供手动操作指导，只禁止 Skill 声称自行执行",
|
||
);
|
||
console.log("✓ Skill 能力边界只拦截执行承诺，不误伤审阅类任务");
|
||
|
||
assert.equal(
|
||
isSuggestionWithinNodeScope(
|
||
"trigger",
|
||
"当用户需要把会议记录整理成行动项时触发",
|
||
),
|
||
true,
|
||
);
|
||
assert.equal(
|
||
isSuggestionWithinNodeScope(
|
||
"trigger",
|
||
"输出使用 Markdown 表格并包含责任人字段",
|
||
),
|
||
false,
|
||
"触发条件阶段不得推荐输出格式内容",
|
||
);
|
||
assert.equal(
|
||
isSuggestionWithinNodeScope(
|
||
"output",
|
||
"新增一个资源推荐模块并给出部署方案",
|
||
),
|
||
false,
|
||
"不得推荐五个节点以外的额外模块或部署方案",
|
||
);
|
||
console.log("✓ 推荐内容严格归属于当前五节点之一");
|
||
|
||
assert.equal(
|
||
evaluateSkillNodeQuality("steps", "分析内容并生成结果").complete,
|
||
false,
|
||
"没有明确顺序和多个动作的执行步骤不得完成",
|
||
);
|
||
assert.equal(
|
||
evaluateSkillNodeQuality(
|
||
"steps",
|
||
"先检查输入，再提取关键信息，然后生成结果并复核。",
|
||
).complete,
|
||
true,
|
||
"满足确定性标准的执行步骤应当完成",
|
||
);
|
||
console.log("✓ 节点完成度由确定性质量标准复核");
|
||
|
||
const initialSuggestions = await jsonRequest(
|
||
"/api/skills/builder/suggestions",
|
||
{
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({
|
||
skillName,
|
||
skillDescription,
|
||
nodes,
|
||
}),
|
||
},
|
||
);
|
||
assert.equal(
|
||
initialSuggestions.source,
|
||
"demo",
|
||
"无 AI 配置时必须明确标记为演示建议",
|
||
);
|
||
assert.equal(
|
||
initialSuggestions.nodeKey,
|
||
"trigger",
|
||
"推荐结果必须明确声明所属节点",
|
||
);
|
||
assert.ok(
|
||
initialSuggestions.suggestions.length >= 2 &&
|
||
initialSuggestions.suggestions.length <= 4,
|
||
"节点进入时必须返回 2-4 条可直接发送的建议",
|
||
);
|
||
assert.equal(
|
||
initialSuggestions.suggestions.some(
|
||
(suggestion) =>
|
||
findUnsupportedSkillCapabilities(suggestion).length > 0 ||
|
||
!isSuggestionWithinNodeScope("trigger", suggestion),
|
||
),
|
||
false,
|
||
"节点推荐不得越出当前节点或包含不支持的能力",
|
||
);
|
||
console.log("✓ 节点进入建议有明确来源和严格结构");
|
||
|
||
const guidanceTurn = await builderTurn("有哪些典型任务");
|
||
assert.equal(
|
||
guidanceTurn.evaluation.intent,
|
||
"request_guidance",
|
||
"向 AI 索要示例时必须识别为求助意图",
|
||
);
|
||
assert.equal(
|
||
guidanceTurn.evaluation.updatedNodeKeys.length,
|
||
0,
|
||
"求助问题不得写入任何 Skill 节点",
|
||
);
|
||
assert.equal(
|
||
nodes.filter((node) => node.completed).length,
|
||
0,
|
||
"求助问题不得完成当前节点",
|
||
);
|
||
assert.match(
|
||
guidanceTurn.reply,
|
||
/例如|比如|常见/,
|
||
"AI 应先提供示例或解释，再继续追问",
|
||
);
|
||
console.log("✓ 求助/索要示例不会写入或完成 Skill 节点");
|
||
|
||
const unsupportedTurn = await builderTurn(
|
||
"执行步骤：调用外部 API 获取实时数据，再运行 Python 脚本生成报告。",
|
||
);
|
||
assert.equal(
|
||
unsupportedTurn.evaluation.intent,
|
||
"request_guidance",
|
||
"超出工作台能力边界的要求必须转为引导",
|
||
);
|
||
assert.deepEqual(
|
||
unsupportedTurn.evaluation.updatedNodeKeys,
|
||
[],
|
||
"脚本执行和外部 API 调用不得写入 Skill 节点",
|
||
);
|
||
assert.match(
|
||
unsupportedTurn.reply,
|
||
/不支持|用户.*提供|手动/,
|
||
"必须解释限制并提供对话内可执行的替代方案",
|
||
);
|
||
console.log("✓ Builder 拒绝脚本/API 执行能力并给出可行替代");
|
||
|
||
const incompleteTurn = await builderTurn("做一套清单");
|
||
assert.equal(
|
||
nodes.filter((node) => node.completed).length,
|
||
0,
|
||
"信息不足时节点不能提前完成",
|
||
);
|
||
assert.match(incompleteTurn.reply, /[？?]/, "信息不足时 AI 必须继续追问");
|
||
|
||
const multiNodeTurn = await builderTurn(
|
||
[
|
||
"触发条件：当用户需要把产品需求整理成验收清单时触发，不处理闲聊。",
|
||
"输入参数：必填产品需求和目标用户，可选优先级、平台与发布日期。",
|
||
"执行步骤：先拆目标，再识别用户路径，然后生成正常、异常和边界场景。",
|
||
"约束与测试：不得臆测未给出的业务规则，每条必须可独立执行并包含反例。",
|
||
].join("；"),
|
||
);
|
||
assert.deepEqual(
|
||
multiNodeTurn.evaluation.updatedNodeKeys,
|
||
["trigger", "inputs", "steps", "constraints"],
|
||
"一条连贯消息涵盖多个节点时，AI 必须一次归类全部内容",
|
||
);
|
||
assert.equal(
|
||
nodes.filter((node) => node.completed).length,
|
||
3,
|
||
"第 4 节点缺失时，只能连续完成第 1-3 节点",
|
||
);
|
||
assert.equal(
|
||
nodes.find((node) => node.key === "constraints")?.ready,
|
||
true,
|
||
"第 5 节点的合格内容必须保留为待解锁",
|
||
);
|
||
assert.equal(
|
||
nodes.find((node) => node.key === "constraints")?.completed,
|
||
false,
|
||
"存在第 4 节点缺口时不得跳到第 5 节点",
|
||
);
|
||
|
||
const filledGap = await builderTurn(
|
||
"输出格式：输出 Markdown 表格，字段为编号、前置条件、操作、预期结果。",
|
||
);
|
||
assert.deepEqual(
|
||
filledGap.evaluation.updatedNodeKeys,
|
||
["output"],
|
||
"补充缺口时 AI 应更新第 4 节点",
|
||
);
|
||
assert.equal(
|
||
nodes.filter((node) => node.completed).length,
|
||
5,
|
||
"第 4 节点补齐后，已准备好的第 5 节点必须自动完成",
|
||
);
|
||
assert.doesNotMatch(
|
||
filledGap.reply,
|
||
/约束与测试[^。]*[？?]/,
|
||
"全部节点自动完成后不得继续追问已经解锁的节点",
|
||
);
|
||
console.log("✓ 一轮可完成多个连续节点，缺口补齐后自动解锁后续节点");
|
||
|
||
assert.deepEqual(
|
||
getSkillNodeQualityIssues(nodes),
|
||
[],
|
||
"全部完成的五个节点必须通过发布质量标准",
|
||
);
|
||
const lowQualityNodes = nodes.map((node) =>
|
||
node.key === "steps"
|
||
? { ...node, content: "分析内容并生成结果", completed: true }
|
||
: node,
|
||
);
|
||
const lowQualitySaveResponse = await fetch(`${baseUrl}/api/skills`, {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({
|
||
name: `${skillName}质量测试`,
|
||
description: skillDescription,
|
||
nodes: lowQualityNodes,
|
||
status: "active",
|
||
}),
|
||
});
|
||
assert.equal(
|
||
lowQualitySaveResponse.status,
|
||
409,
|
||
"即使前端标记 completed，低质量节点也不得发布",
|
||
);
|
||
assert.match(
|
||
(await lowQualitySaveResponse.json()).error,
|
||
/尚未达到可发布标准|明确顺序/,
|
||
);
|
||
console.log("✓ 保存接口独立复核五节点质量");
|
||
|
||
const duplicateNodeResponse = await fetch(`${baseUrl}/api/skills`, {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({
|
||
name: `${skillName}重复节点测试`,
|
||
description: skillDescription,
|
||
nodes: nodes.map((node) => ({ ...node, key: "trigger" })),
|
||
status: "active",
|
||
}),
|
||
});
|
||
assert.equal(
|
||
duplicateNodeResponse.status,
|
||
400,
|
||
"五个重复节点不得绕过固定流程校验",
|
||
);
|
||
assert.match(
|
||
JSON.stringify(await duplicateNodeResponse.json()),
|
||
/必须且只能包含/,
|
||
);
|
||
console.log("✓ 保存接口强制五种节点各一个");
|
||
|
||
const unsupportedNodes = nodes.map((node) =>
|
||
node.key === "steps"
|
||
? {
|
||
...node,
|
||
content: "调用外部 API 获取实时数据，再运行 Python 脚本生成报告。",
|
||
}
|
||
: node,
|
||
);
|
||
assert.ok(
|
||
getSkillCapabilityViolations({
|
||
name: skillName,
|
||
description: skillDescription,
|
||
nodes: unsupportedNodes,
|
||
}).length >= 2,
|
||
"保存前能力校验必须定位违规节点",
|
||
);
|
||
const unsupportedSaveResponse = await fetch(`${baseUrl}/api/skills`, {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({
|
||
name: `${skillName}越界测试`,
|
||
description: skillDescription,
|
||
nodes: unsupportedNodes,
|
||
status: "active",
|
||
}),
|
||
});
|
||
const unsupportedSavePayload = await unsupportedSaveResponse.json();
|
||
assert.equal(
|
||
unsupportedSaveResponse.status,
|
||
409,
|
||
"绕过 Builder 直接保存越界 Skill 时也必须拒绝",
|
||
);
|
||
assert.match(unsupportedSavePayload.error, /不支持.*脚本.*API/);
|
||
console.log("✓ 保存接口拒绝包含脚本/API 执行承诺的 Skill");
|
||
|
||
const created = await jsonRequest("/api/skills", {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({
|
||
name: skillName,
|
||
description: skillDescription,
|
||
nodes,
|
||
status: "active",
|
||
}),
|
||
});
|
||
const skillId = created.skill.id;
|
||
assert.ok(skillId, "必须自动创建 Skill");
|
||
console.log("✓ 自动创建 Skill");
|
||
|
||
const editedTurn = await builderTurn(
|
||
"补充规则：当需求只有一句话时，先追问目标用户再生成清单。",
|
||
);
|
||
assert.ok(
|
||
editedTurn.evaluation.updatedNodeKeys.includes("trigger"),
|
||
"编辑时也必须由 AI 自动判断目标节点",
|
||
);
|
||
await jsonRequest(`/api/skills/${skillId}`, {
|
||
method: "PUT",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({
|
||
name: skillName,
|
||
description: skillDescription,
|
||
nodes,
|
||
status: "active",
|
||
}),
|
||
});
|
||
const edited = await jsonRequest(`/api/skills/${skillId}`);
|
||
assert.match(edited.skill.nodes[0].content, /目标用户/);
|
||
console.log("✓ 只能通过 Builder 对话并由 AI 自动归类编辑 Skill 内容");
|
||
|
||
const chat = await jsonRequest("/api/chat");
|
||
assert.equal(chat.conversation.title, "", "新会话默认标题必须为空");
|
||
|
||
const firstChatClientRequestId = `acceptance_${crypto.randomUUID()}`;
|
||
const firstChatRequestBody = {
|
||
conversationId: chat.conversation.id,
|
||
message: "产品需求 验收清单 输入参数 输出格式，请帮我整理。",
|
||
selectedSkillIds: [skillId],
|
||
clientRequestId: firstChatClientRequestId,
|
||
};
|
||
const runPayload = await jsonRequest("/api/chat", {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify(firstChatRequestBody),
|
||
});
|
||
const runId = runPayload.run.id;
|
||
|
||
const firstResponse = await fetch(
|
||
`${baseUrl}/api/chat/runs/${runId}/stream?after=0`,
|
||
);
|
||
const firstEvents = await readFirstSseBatch(firstResponse);
|
||
const cursor = Math.max(...firstEvents.map((event) => event.seq ?? 0));
|
||
assert.ok(cursor > 0, "断连前必须拿到可恢复序号");
|
||
|
||
await new Promise((resolve) => setTimeout(resolve, 180));
|
||
const resumedResponse = await fetch(
|
||
`${baseUrl}/api/chat/runs/${runId}/stream?after=${cursor}`,
|
||
);
|
||
const resumedEvents = await readSse(resumedResponse);
|
||
const allEvents = [...firstEvents, ...resumedEvents];
|
||
const sequences = allEvents.map((event) => event.seq).filter(Boolean);
|
||
assert.equal(new Set(sequences).size, sequences.length, "重连不得重复事件");
|
||
const usage = allEvents.find((event) => event.type === "skill_usage");
|
||
assert.deepEqual(usage?.skillIds, [skillId], "必须反馈实际用到的 Skill");
|
||
assert.equal(usage?.source, "keyword_fallback");
|
||
assert.ok(usage?.reason, "Skill 使用事件必须记录判断来源和原因");
|
||
const usageSummary = allEvents
|
||
.filter((event) => event.type === "usage")
|
||
.at(-1)?.usage;
|
||
assert.equal(
|
||
usageSummary?.status,
|
||
"demo",
|
||
"本地演示回答必须明确标记为零费用演示用量",
|
||
);
|
||
assert.equal(usageSummary?.totalTokens, 0);
|
||
assert.equal(usageSummary?.estimatedCostMicros, 0);
|
||
assert.ok(
|
||
usageSummary?.durationMs >= 0,
|
||
"每轮回答必须记录端到端用时",
|
||
);
|
||
const completedRetention = allEvents.find(
|
||
(event) => event.type === "skill_retention",
|
||
);
|
||
assert.equal(
|
||
completedRetention?.state,
|
||
"complete",
|
||
"任务已交付完成时必须清空 Skill",
|
||
);
|
||
assert.deepEqual(completedRetention?.skillIds, []);
|
||
assert.ok(
|
||
allEvents.some((event) => event.type === "complete"),
|
||
"聊天必须完成",
|
||
);
|
||
assert.ok(
|
||
allEvents.some(
|
||
(event) => event.type === "conversation_title" && event.title,
|
||
),
|
||
"第一轮结束后必须自动生成标题",
|
||
);
|
||
const generatedTitle = allEvents.find(
|
||
(event) => event.type === "conversation_title",
|
||
)?.title;
|
||
assert.equal(
|
||
generatedTitle,
|
||
"产品需求与验收梳理",
|
||
"标题必须提炼主题，不能直接截断用户原句",
|
||
);
|
||
assert.ok(generatedTitle.length <= 18, "自动标题不得超过 18 个字符");
|
||
const chatAfterMeasuredRun = await jsonRequest("/api/chat");
|
||
const persistedUsage = chatAfterMeasuredRun.messages.find(
|
||
(message) => message.id === runPayload.run.assistantMessageId,
|
||
)?.usage;
|
||
assert.equal(
|
||
persistedUsage?.status,
|
||
"demo",
|
||
"刷新后必须恢复回答的 Token 与费用摘要",
|
||
);
|
||
assert.equal(
|
||
persistedUsage?.durationMs,
|
||
usageSummary.durationMs,
|
||
"刷新后必须恢复本轮用时",
|
||
);
|
||
const messagesBeforeIdempotentRetry =
|
||
chatAfterMeasuredRun.messages.length;
|
||
const idempotentRetry = await jsonRequest("/api/chat", {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify(firstChatRequestBody),
|
||
});
|
||
assert.equal(
|
||
idempotentRetry.run.id,
|
||
runPayload.run.id,
|
||
"相同 clientRequestId 重试必须返回原 Run",
|
||
);
|
||
assert.equal(
|
||
(await jsonRequest("/api/chat")).messages.length,
|
||
messagesBeforeIdempotentRetry,
|
||
"幂等重试不得重复写入消息",
|
||
);
|
||
console.log("✓ Skill 选择、实际调用反馈与 SSE 断线续传");
|
||
|
||
const unusedSkillPayload = await jsonRequest("/api/chat", {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({
|
||
conversationId: chat.conversation.id,
|
||
message: "请写一句关于春天的问候。",
|
||
selectedSkillIds: [skillId],
|
||
clientRequestId: `acceptance_unused_skill_${crypto.randomUUID()}`,
|
||
}),
|
||
});
|
||
const unusedSkillEvents = await readSse(
|
||
await fetch(
|
||
`${baseUrl}/api/chat/runs/${unusedSkillPayload.run.id}/stream?after=0`,
|
||
),
|
||
);
|
||
const unusedSkillUsage = unusedSkillEvents.find(
|
||
(event) => event.type === "skill_usage",
|
||
);
|
||
assert.deepEqual(
|
||
unusedSkillUsage?.skillIds,
|
||
[],
|
||
"不相关任务不得调用已选择的 Skill",
|
||
);
|
||
const unusedSkillStatuses = unusedSkillEvents
|
||
.filter((event) => event.type === "status")
|
||
.map((event) => event.label);
|
||
assert.ok(
|
||
unusedSkillStatuses.includes("正在思考"),
|
||
"未调用 Skill 时应使用普通思考状态",
|
||
);
|
||
assert.equal(
|
||
unusedSkillStatuses.some((label) => /无需调用|未使用 Skill/.test(label)),
|
||
false,
|
||
"未调用 Skill 时不得展示内部路由结论",
|
||
);
|
||
console.log("✓ 未调用 Skill 时只展示普通思考状态");
|
||
|
||
const staleUserMessageId = `stale_user_${crypto.randomUUID()}`;
|
||
const staleAssistantMessageId = `stale_assistant_${crypto.randomUUID()}`;
|
||
const staleRunId = `stale_run_${crypto.randomUUID()}`;
|
||
const staleDatabase = new DatabaseSync(databasePath);
|
||
staleDatabase.exec("PRAGMA busy_timeout = 5000;");
|
||
const insertSyntheticMessage = staleDatabase.prepare(`
|
||
INSERT INTO messages
|
||
(id, conversation_id, role, content, selected_skill_ids,
|
||
used_skill_ids, status, error_message, usage_json, created_at)
|
||
VALUES (?, ?, ?, ?, '[]', '[]', ?, '', '', ?)
|
||
`);
|
||
insertSyntheticMessage.run(
|
||
staleUserMessageId,
|
||
chat.conversation.id,
|
||
"user",
|
||
"模拟服务重启前的用户消息",
|
||
"complete",
|
||
"2026-01-01T00:00:00.000Z",
|
||
);
|
||
insertSyntheticMessage.run(
|
||
staleAssistantMessageId,
|
||
chat.conversation.id,
|
||
"assistant",
|
||
"",
|
||
"streaming",
|
||
"2026-01-01T00:00:00.001Z",
|
||
);
|
||
staleDatabase
|
||
.prepare(
|
||
`INSERT INTO chat_runs
|
||
(id, client_request_id, conversation_id, user_message_id,
|
||
assistant_message_id, status, cancel_requested, created_at, updated_at)
|
||
VALUES (?, ?, ?, ?, ?, 'running', 0, ?, ?)`,
|
||
)
|
||
.run(
|
||
staleRunId,
|
||
`stale_client_${crypto.randomUUID()}`,
|
||
chat.conversation.id,
|
||
staleUserMessageId,
|
||
staleAssistantMessageId,
|
||
"2026-01-01T00:00:00.000Z",
|
||
"2026-01-01T00:00:00.000Z",
|
||
);
|
||
staleDatabase.close();
|
||
const staleRunEvents = await readSse(
|
||
await fetch(`${baseUrl}/api/chat/runs/${staleRunId}/stream?after=0`),
|
||
);
|
||
assert.ok(
|
||
staleRunEvents.some(
|
||
(event) =>
|
||
event.type === "error" && /服务重启|超时/.test(event.message),
|
||
),
|
||
"孤儿 Run 必须自动进入错误终态并关闭 SSE",
|
||
);
|
||
const chatAfterStaleRun = await jsonRequest("/api/chat");
|
||
assert.equal(
|
||
chatAfterStaleRun.messages.find(
|
||
(message) => message.id === staleAssistantMessageId,
|
||
)?.status,
|
||
"error",
|
||
"孤儿 Run 对应的助手消息必须标记失败",
|
||
);
|
||
assert.equal(chatAfterStaleRun.continuation.state, "complete");
|
||
console.log("✓ 服务重启或超时遗留的孤儿 Run 会自动回收");
|
||
|
||
const seededSkills = await jsonRequest("/api/skills");
|
||
const sqlReviewSkill = seededSkills.skills.find(
|
||
(skill) => skill.name === "SQL 安全审阅",
|
||
);
|
||
assert.ok(sqlReviewSkill, "预置的 SQL 安全审阅 Skill 必须存在");
|
||
const explainSkillPayload = await jsonRequest("/api/chat", {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({
|
||
conversationId: chat.conversation.id,
|
||
message: "这个 Skill 是干嘛的？",
|
||
selectedSkillIds: [sqlReviewSkill.id],
|
||
clientRequestId: `acceptance_explain_skill_${crypto.randomUUID()}`,
|
||
}),
|
||
});
|
||
const explainSkillEvents = await readSse(
|
||
await fetch(
|
||
`${baseUrl}/api/chat/runs/${explainSkillPayload.run.id}/stream?after=0`,
|
||
),
|
||
);
|
||
const explainSkillUsage = explainSkillEvents.find(
|
||
(event) => event.type === "skill_usage",
|
||
);
|
||
const explainSkillContent = explainSkillEvents
|
||
.filter((event) => event.type === "token")
|
||
.map((event) => event.token)
|
||
.join("");
|
||
assert.deepEqual(
|
||
explainSkillUsage?.skillIds,
|
||
[sqlReviewSkill.id],
|
||
"询问已选 Skill 的用途时必须使用该 Skill 的真实资料",
|
||
);
|
||
assert.match(explainSkillContent, /检查 SQL 的正确性、性能风险与数据安全边界/);
|
||
assert.match(explainSkillContent, /当用户提供 SQL、查询计划或数据库变更/);
|
||
assert.match(explainSkillContent, /任何 DELETE、UPDATE、DDL 建议必须附回滚方案/);
|
||
assert.doesNotMatch(
|
||
explainSkillContent,
|
||
/报错弹窗|缺少 DLL|代理配置错误/,
|
||
"Skill 介绍不得串用其他 Skill 的内容",
|
||
);
|
||
console.log("✓ 询问 Skill 用途时严格返回已选 Skill 的真实定义");
|
||
|
||
const followUpPayload = await jsonRequest("/api/chat", {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({
|
||
conversationId: chat.conversation.id,
|
||
message:
|
||
"产品需求 验收清单：我还没提供具体内容，请先询问我需要补充的信息。",
|
||
selectedSkillIds: [skillId],
|
||
clientRequestId: `acceptance_follow_up_${crypto.randomUUID()}`,
|
||
}),
|
||
});
|
||
const followUpEvents = await readSse(
|
||
await fetch(
|
||
`${baseUrl}/api/chat/runs/${followUpPayload.run.id}/stream?after=0`,
|
||
),
|
||
);
|
||
const followUpRetention = followUpEvents.find(
|
||
(event) => event.type === "skill_retention",
|
||
);
|
||
assert.equal(
|
||
followUpRetention?.state,
|
||
"awaiting_input",
|
||
"AI 要求用户补充必要信息时，任务必须保持继续状态",
|
||
);
|
||
assert.deepEqual(
|
||
followUpRetention?.skillIds,
|
||
[skillId],
|
||
"未完成的多轮任务必须保留实际使用的 Skill",
|
||
);
|
||
const chatWithRetainedSkill = await jsonRequest("/api/chat");
|
||
assert.deepEqual(
|
||
chatWithRetainedSkill.retainedSkillIds,
|
||
[skillId],
|
||
"刷新页面后仍必须恢复待继续任务的 Skill",
|
||
);
|
||
assert.equal(
|
||
chatWithRetainedSkill.continuation.state,
|
||
"awaiting_input",
|
||
"刷新后必须恢复显式续接状态",
|
||
);
|
||
assert.ok(
|
||
chatWithRetainedSkill.continuation.expectedInput,
|
||
"续接状态必须记录下一轮期待用户补充的内容",
|
||
);
|
||
assert.equal(
|
||
chatWithRetainedSkill.continuation.source,
|
||
"deterministic",
|
||
"明确追问应由确定性规则识别，不依赖 AI 猜测",
|
||
);
|
||
console.log("✓ AI 追问时跨轮保留 Skill，并持久化到 SQLite");
|
||
|
||
const continuedPayload = await jsonRequest("/api/chat", {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({
|
||
conversationId: chat.conversation.id,
|
||
message: "需要",
|
||
selectedSkillIds: [skillId],
|
||
clientRequestId: `acceptance_continued_skill_${crypto.randomUUID()}`,
|
||
}),
|
||
});
|
||
const continuedEvents = await readSse(
|
||
await fetch(
|
||
`${baseUrl}/api/chat/runs/${continuedPayload.run.id}/stream?after=0`,
|
||
),
|
||
);
|
||
const continuedUsage = continuedEvents.find(
|
||
(event) => event.type === "skill_usage",
|
||
);
|
||
assert.deepEqual(
|
||
continuedUsage?.skillIds,
|
||
[skillId],
|
||
"用户用“需要”等短回复接受上一轮提议时必须继续使用保留的 Skill",
|
||
);
|
||
assert.equal(
|
||
continuedUsage?.source,
|
||
"continuation",
|
||
"省略回复必须通过显式续接状态恢复 Skill",
|
||
);
|
||
console.log("✓ “需要”等省略回复会继续使用上一轮保留的 Skill");
|
||
|
||
const stopPayload = await jsonRequest("/api/chat", {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({
|
||
conversationId: chat.conversation.id,
|
||
message: "请生成一份很长的补充清单。",
|
||
selectedSkillIds: [],
|
||
clientRequestId: `acceptance_stop_${crypto.randomUUID()}`,
|
||
}),
|
||
});
|
||
const conflictResponse = await fetch(`${baseUrl}/api/chat`, {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({
|
||
conversationId: chat.conversation.id,
|
||
message: "这条并发冲突消息不应写入数据库。",
|
||
selectedSkillIds: [],
|
||
clientRequestId: `acceptance_conflict_${crypto.randomUUID()}`,
|
||
}),
|
||
});
|
||
assert.equal(
|
||
conflictResponse.status,
|
||
409,
|
||
"同一会话同时只能创建一个活动 Run",
|
||
);
|
||
const visibleStoppedContent = "客户端点击中止时已经显示的内容";
|
||
await jsonRequest(`/api/chat/runs/${stopPayload.run.id}/stop`, {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({ visibleContent: visibleStoppedContent }),
|
||
});
|
||
const stoppedEvents = await readSse(
|
||
await fetch(
|
||
`${baseUrl}/api/chat/runs/${stopPayload.run.id}/stream?after=0`,
|
||
),
|
||
);
|
||
assert.ok(
|
||
stoppedEvents.some((event) => event.type === "aborted"),
|
||
"中止必须作为终态被记录",
|
||
);
|
||
const chatAfterStop = await jsonRequest("/api/chat");
|
||
const stoppedMessage = chatAfterStop.messages.find(
|
||
(message) => message.id === stopPayload.run.assistantMessageId,
|
||
);
|
||
assert.equal(stoppedMessage?.status, "stopped");
|
||
assert.equal(
|
||
stoppedMessage?.content,
|
||
visibleStoppedContent,
|
||
"中止后只保留用户点击时已经看到的内容",
|
||
);
|
||
assert.ok(
|
||
stoppedMessage?.usage?.durationMs >= 0,
|
||
"中止回答也必须记录端到端用时",
|
||
);
|
||
assert.equal(
|
||
chatAfterStop.messages.some(
|
||
(message) => message.content === "这条并发冲突消息不应写入数据库。",
|
||
),
|
||
false,
|
||
"并发冲突事务不得留下孤立消息",
|
||
);
|
||
console.log("✓ 中止被视为一轮完成");
|
||
|
||
const renamed = await jsonRequest("/api/conversation", {
|
||
method: "PATCH",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({ title: "验收测试会话" }),
|
||
});
|
||
assert.equal(renamed.conversation.title, "验收测试会话");
|
||
const reset = await jsonRequest("/api/conversation", { method: "DELETE" });
|
||
assert.equal(reset.conversation.title, "");
|
||
const resetChat = await jsonRequest("/api/chat");
|
||
assert.equal(resetChat.messages.length, 0);
|
||
assert.equal(resetChat.continuation.state, "complete");
|
||
assert.deepEqual(resetChat.continuation.skillIds, []);
|
||
console.log("✓ 标题编辑与会话重置");
|
||
|
||
await fetch(`${baseUrl}/api/skills/${skillId}`, { method: "DELETE" }).then(
|
||
(response) => assert.equal(response.status, 204),
|
||
);
|
||
const skillsAfterDelete = await jsonRequest("/api/skills");
|
||
assert.equal(
|
||
skillsAfterDelete.skills.some((skill) => skill.id === skillId),
|
||
false,
|
||
);
|
||
console.log("✓ 删除 Skill 并刷新列表");
|
||
}
|
||
|
||
try {
|
||
await runAcceptance();
|
||
console.log("\n验收通过：创建、编辑、删除、选择、调用反馈、中止、标题与重连均正常。");
|
||
} finally {
|
||
server.kill("SIGTERM");
|
||
await new Promise((resolve) => {
|
||
const timeout = setTimeout(resolve, 2_000);
|
||
server.once("exit", () => {
|
||
clearTimeout(timeout);
|
||
resolve();
|
||
});
|
||
});
|
||
rmSync(tempDir, { recursive: true, force: true });
|
||
}
|