242 lines
12 KiB
JavaScript
242 lines
12 KiB
JavaScript
|
/**
|
|||
|
|
* KB Prompt A/B 对比测试
|
|||
|
|
* 用失败用例验证优化后的prompt效果
|
|||
|
|
* 零依赖：仅用Node内置模块
|
|||
|
|
*/
|
|||
|
|
const path = require('path');
|
|||
|
|
const https = require('https');
|
|||
|
|
const fs = require('fs');
|
|||
|
|
|
|||
|
|
// 手动加载 .env
|
|||
|
|
const envPath = path.join(__dirname, '../.env');
|
|||
|
|
if (fs.existsSync(envPath)) {
|
|||
|
|
fs.readFileSync(envPath, 'utf8').split('\n').forEach(line => {
|
|||
|
|
const trimmed = line.trim();
|
|||
|
|
if (!trimmed || trimmed.startsWith('#')) return;
|
|||
|
|
const idx = trimmed.indexOf('=');
|
|||
|
|
if (idx > 0) {
|
|||
|
|
const key = trimmed.slice(0, idx).trim();
|
|||
|
|
let val = trimmed.slice(idx + 1).trim();
|
|||
|
|
if ((val.startsWith('"') && val.endsWith('"')) || (val.startsWith("'") && val.endsWith("'"))) {
|
|||
|
|
val = val.slice(1, -1);
|
|||
|
|
}
|
|||
|
|
if (!process.env[key]) process.env[key] = val;
|
|||
|
|
}
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const kbHttpAgent = new https.Agent({
|
|||
|
|
keepAlive: true,
|
|||
|
|
keepAliveMsecs: 30000,
|
|||
|
|
maxSockets: 6,
|
|||
|
|
timeout: 15000,
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
// ===== 旧prompt =====
|
|||
|
|
const OLD_PROMPT = '你是大沃的智能助手，负责回答与PM-FitLine德国产品相关的问题。产品用法、成分、剂量、价格等专业信息必须依据知识库，不得自行补充或猜测；公司背景、理念等常识性内容可适当补充。不得编造不存在的产品名称或数据。若知识库无相关内容，坦诚说明并建议查看产品说明书或咨询推荐人。回答口语化、简洁、专业，1-3句给结论，150字内。';
|
|||
|
|
|
|||
|
|
// ===== 新prompt =====
|
|||
|
|
const NEW_PROMPT = '你是大沃的智能助手，负责回答与PM-FitLine德国产品相关的问题。知识库涵盖近50款PM-FitLine产品的完整资料（成分、用法、剂量、价格、规格、搭配方案、好转反应等）及117个常见问答。回答规则：产品相关具体信息必须严格依据知识库，不得猜测或自行补充；公司背景、健康常识可适当补充。产品常有别名（小红=艾特维、大白=倍适、小白=维适多等），请注意识别。不得编造产品名或数据。PM是营养品非药物，涉及疾病建议咨询医生。若知识库无相关内容，坦诚说明并建议咨询推荐人。回答口语化、简洁，1-3句给结论，150字内。';
|
|||
|
|
|
|||
|
|
// ===== 全面测试用例：10个维度 =====
|
|||
|
|
const TEST_CASES = [
|
|||
|
|
// ── 1. 产品用法（营养品） ──
|
|||
|
|
{ cat: '用法', name: '细胞抗氧素怎么吃(原失败)', query: 'Apple Antioxy Zellschutz 细胞抗氧素 怎么吃', original: '细胞抗氧素怎么吃', expectHit: true },
|
|||
|
|
{ cat: '用法', name: '小红怎么喝', query: 'Activize Oxyplus 艾特维 小红 怎么喝', original: '小红怎么喝', expectHit: true },
|
|||
|
|
{ cat: '用法', name: '小白什么时候喝', query: 'Restorate 维适多 小白 什么时候喝 服用方法', original: '小白什么时候喝', expectHit: true },
|
|||
|
|
{ cat: '用法', name: 'Q10怎么用', query: 'Q10辅酵素 怎么用 用法', original: 'Q10怎么用', expectHit: true },
|
|||
|
|
{ cat: '用法', name: '心脏衰竭者小红用量', query: 'PM-Fitline 心脏衰竭 小红 用量 怎么喝', original: '心脏衰竭的人小红怎么喝', expectHit: true },
|
|||
|
|
|
|||
|
|
// ── 2. 产品价格 ──
|
|||
|
|
{ cat: '价格', name: '大白多少钱', query: 'Basics 倍适 大白 价格 多少钱', original: '大白多少钱', expectHit: true },
|
|||
|
|
{ cat: '价格', name: '小绿排毒饮价格', query: 'D-Drink 小绿 排毒饮 价格 多少钱', original: '小绿多少钱', expectHit: true },
|
|||
|
|
{ cat: '价格', name: '祛皱凝胶价格', query: 'Ultimate Young 三分钟瞬间祛皱凝胶 价格', original: '祛皱凝胶多少钱', expectHit: true },
|
|||
|
|
|
|||
|
|
// ── 3. 产品成分 ──
|
|||
|
|
{ cat: '成分', name: '小红里有什么成分', query: 'Activize Oxyplus 艾特维 小红 成分 配方', original: '小红里面有什么成分', expectHit: true },
|
|||
|
|
{ cat: '成分', name: '美容饮成分', query: 'Beauty 肽美 美容饮 成分', original: '美容饮有什么成分', expectHit: true },
|
|||
|
|
|
|||
|
|
// ── 4. 产品搭配 ──
|
|||
|
|
{ cat: '搭配', name: '基础三合一是什么', query: '德国PM细胞营养素 基础套装 大白 小红 小白', original: '基础三合一是什么', expectHit: true },
|
|||
|
|
{ cat: '搭配', name: '减肥搭配方案', query: 'TopShape ProShape 减肥 搭配 方案', original: '想减肥应该吃什么搭配', expectHit: true },
|
|||
|
|
|
|||
|
|
// ── 5. 好转反应 ──
|
|||
|
|
{ cat: '好转反应', name: '为什么会痒', query: 'PM-Fitline 好转反应 痒 发红', original: '喝了PM为什么会痒', expectHit: true },
|
|||
|
|
{ cat: '好转反应', name: '喝了头晕怎么回事', query: 'PM-Fitline 好转反应 头晕 目眩', original: '喝完营养素头晕怎么回事', expectHit: true },
|
|||
|
|
{ cat: '好转反应', name: '为什么喝小红胃痛', query: 'PM-Fitline 小红 艾特维 胃痛 好转反应', original: '为什么喝小红胃会痛', expectHit: true },
|
|||
|
|
|
|||
|
|
// ── 6. 别名识别 ──
|
|||
|
|
{ cat: '别名', name: '小黑是什么产品', query: 'MEN+ 倍力健 小黑 产品介绍', original: '小黑是什么', expectHit: true },
|
|||
|
|
{ cat: '别名', name: '乐活50+适合谁', query: 'Generation 50+ 乐活50+ 适合谁 适用人群', original: '乐活50+适合谁吃', expectHit: true },
|
|||
|
|
|
|||
|
|
// ── 7. 护肤品 ──
|
|||
|
|
{ cat: '护肤', name: '眼霜怎么用', query: 'Eye Care 全效眼霜 用法 怎么用', original: '眼霜怎么用', expectHit: true },
|
|||
|
|
{ cat: '护肤', name: '面膜多久敷一次', query: 'Hydrating-Shot Mask 靓白保湿面膜 多久 频率', original: '面膜多久敷一次', expectHit: true },
|
|||
|
|
|
|||
|
|
// ── 8. 117问答/疾病相关 ──
|
|||
|
|
{ cat: 'Q&A', name: '高血压能吃PM吗', query: 'PM-Fitline 高血压 能吃吗 影响', original: '高血压能吃PM吗', expectHit: true },
|
|||
|
|
{ cat: 'Q&A', name: '糖尿病能减药吗', query: 'PM-Fitline 糖尿病 减药 停药', original: '吃PM后糖尿病的药能减吗', expectHit: true },
|
|||
|
|
{ cat: 'Q&A', name: '小红咖啡因和咖啡一样吗', query: 'PM-Fitline 小红 艾特维 咖啡因 瓜拉纳 区别', original: '小红里的咖啡因跟咖啡一样吗', expectHit: true },
|
|||
|
|
|
|||
|
|
// ── 9. 边界测试（KB中不存在的信息） ──
|
|||
|
|
{ cat: '边界', name: '不存在的产品', query: 'PM-Fitline 超级修复胶囊', original: 'PM有超级修复胶囊这个产品吗', expectHit: false },
|
|||
|
|
{ cat: '边界', name: '跟主题无关的问题', query: '今天天气怎么样', original: '今天天气怎么样', expectHit: false },
|
|||
|
|
|
|||
|
|
// ── 10. 公司/系统 ──
|
|||
|
|
{ cat: '公司', name: 'PM是什么公司', query: 'PM-International 德国PM公司 介绍', original: 'PM是什么公司', expectHit: true },
|
|||
|
|
|
|||
|
|
// ── 11. 一成系统 ──
|
|||
|
|
{ cat: '一成系统', name: '一成系统如何赋能德国PM', query: '一成系统 赋能 德国PM', original: '一成系统如何赋能德国PM', expectHit: true },
|
|||
|
|
];
|
|||
|
|
|
|||
|
|
function httpsPost(url, body, headers) {
|
|||
|
|
return new Promise((resolve, reject) => {
|
|||
|
|
const data = JSON.stringify(body);
|
|||
|
|
const parsed = new URL(url);
|
|||
|
|
const req = https.request({
|
|||
|
|
hostname: parsed.hostname,
|
|||
|
|
path: parsed.pathname,
|
|||
|
|
method: 'POST',
|
|||
|
|
headers: { ...headers, 'Content-Length': Buffer.byteLength(data) },
|
|||
|
|
agent: kbHttpAgent,
|
|||
|
|
timeout: 15000,
|
|||
|
|
}, (res) => {
|
|||
|
|
let chunks = [];
|
|||
|
|
res.on('data', c => chunks.push(c));
|
|||
|
|
res.on('end', () => {
|
|||
|
|
try {
|
|||
|
|
resolve(JSON.parse(Buffer.concat(chunks).toString()));
|
|||
|
|
} catch (e) {
|
|||
|
|
reject(new Error('JSON parse error: ' + Buffer.concat(chunks).toString().slice(0, 200)));
|
|||
|
|
}
|
|||
|
|
});
|
|||
|
|
});
|
|||
|
|
req.on('error', reject);
|
|||
|
|
req.on('timeout', () => { req.destroy(); reject(new Error('timeout')); });
|
|||
|
|
req.write(data);
|
|||
|
|
req.end();
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function callKB(systemPrompt, query, label) {
|
|||
|
|
const endpointId = process.env.VOLC_ARK_KNOWLEDGE_ENDPOINT_ID || process.env.VOLC_ARK_ENDPOINT_ID;
|
|||
|
|
const kbModel = process.env.VOLC_ARK_KB_MODEL || endpointId;
|
|||
|
|
const authKey = process.env.VOLC_ARK_API_KEY || process.env.VOLC_ACCESS_KEY_ID;
|
|||
|
|
const kbIds = (process.env.VOLC_ARK_KNOWLEDGE_BASE_IDS || '').split(',').map(id => id.trim()).filter(Boolean);
|
|||
|
|
const topK = parseInt(process.env.VOLC_ARK_KNOWLEDGE_TOP_K) || 3;
|
|||
|
|
const threshold = parseFloat(process.env.VOLC_ARK_KNOWLEDGE_THRESHOLD) || 0.3;
|
|||
|
|
|
|||
|
|
const body = {
|
|||
|
|
model: kbModel,
|
|||
|
|
messages: [
|
|||
|
|
{ role: 'system', content: systemPrompt },
|
|||
|
|
{ role: 'user', content: query },
|
|||
|
|
],
|
|||
|
|
metadata: {
|
|||
|
|
knowledge_base: {
|
|||
|
|
dataset_ids: kbIds,
|
|||
|
|
top_k: topK,
|
|||
|
|
threshold: threshold,
|
|||
|
|
},
|
|||
|
|
},
|
|||
|
|
stream: false,
|
|||
|
|
max_tokens: 80,
|
|||
|
|
thinking: { type: 'disabled' },
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
const start = Date.now();
|
|||
|
|
try {
|
|||
|
|
const res = await httpsPost(
|
|||
|
|
'https://ark.cn-beijing.volces.com/api/v3/chat/completions',
|
|||
|
|
body,
|
|||
|
|
{
|
|||
|
|
'Content-Type': 'application/json',
|
|||
|
|
'Authorization': `Bearer ${authKey}`,
|
|||
|
|
}
|
|||
|
|
);
|
|||
|
|
const elapsed = Date.now() - start;
|
|||
|
|
const content = res?.choices?.[0]?.message?.content || '(empty)';
|
|||
|
|
const usage = res?.usage || {};
|
|||
|
|
const isNoHit = /未找到|未提及|暂无|没有相关|无法|不确定/.test(content);
|
|||
|
|
return { label, content, elapsed, usage, hit: !isNoHit };
|
|||
|
|
} catch (err) {
|
|||
|
|
return { label, content: `ERROR: ${err.message}`, elapsed: Date.now() - start, usage: {}, hit: false };
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function runTest() {
|
|||
|
|
console.log('============================================================');
|
|||
|
|
console.log(' KB 新Prompt 全面对话测试（10维度 x 25用例）');
|
|||
|
|
console.log('============================================================\n');
|
|||
|
|
|
|||
|
|
const kbModel = process.env.VOLC_ARK_KB_MODEL || process.env.VOLC_ARK_KNOWLEDGE_ENDPOINT_ID || '(not set)';
|
|||
|
|
const threshold = process.env.VOLC_ARK_KNOWLEDGE_THRESHOLD || '0.3';
|
|||
|
|
console.log(`模型: ${kbModel} | 阈值: ${threshold} | 用例数: ${TEST_CASES.length}\n`);
|
|||
|
|
|
|||
|
|
const results = [];
|
|||
|
|
const catStats = {};
|
|||
|
|
|
|||
|
|
for (let i = 0; i < TEST_CASES.length; i++) {
|
|||
|
|
const tc = TEST_CASES[i];
|
|||
|
|
console.log(`[${i + 1}/${TEST_CASES.length}] 【${tc.cat}】${tc.name}`);
|
|||
|
|
console.log(` 原始问题: "${tc.original}"`);
|
|||
|
|
|
|||
|
|
const r = await callKB(NEW_PROMPT, tc.query, tc.name);
|
|||
|
|
const matchExpect = r.hit === tc.expectHit;
|
|||
|
|
|
|||
|
|
// 对边界测试特殊处理：no-hit是期望结果
|
|||
|
|
const icon = matchExpect ? '✅' : '⚠️';
|
|||
|
|
const hitLabel = tc.expectHit
|
|||
|
|
? (r.hit ? 'HIT' : 'MISS(应命中)')
|
|||
|
|
: (r.hit ? 'HIT(应未命中)' : 'NO-HIT(正确)');
|
|||
|
|
|
|||
|
|
console.log(` ${icon} ${hitLabel} | ${r.elapsed}ms | tokens: ${r.usage.prompt_tokens || '?'}→${r.usage.completion_tokens || '?'}`);
|
|||
|
|
console.log(` 回答: ${r.content.slice(0, 150)}${r.content.length > 150 ? '...' : ''}\n`);
|
|||
|
|
|
|||
|
|
results.push({ ...tc, ...r, matchExpect });
|
|||
|
|
|
|||
|
|
if (!catStats[tc.cat]) catStats[tc.cat] = { total: 0, correct: 0, totalMs: 0 };
|
|||
|
|
catStats[tc.cat].total++;
|
|||
|
|
if (matchExpect) catStats[tc.cat].correct++;
|
|||
|
|
catStats[tc.cat].totalMs += r.elapsed;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ── 分类汇总 ──
|
|||
|
|
console.log('============================================================');
|
|||
|
|
console.log(' 分类汇总');
|
|||
|
|
console.log('============================================================');
|
|||
|
|
let totalCorrect = 0, totalMs = 0;
|
|||
|
|
for (const [cat, s] of Object.entries(catStats)) {
|
|||
|
|
const pct = Math.round(s.correct / s.total * 100);
|
|||
|
|
const avgMs = Math.round(s.totalMs / s.total);
|
|||
|
|
console.log(` 【${cat}】${s.correct}/${s.total} 正确 (${pct}%) | 平均 ${avgMs}ms`);
|
|||
|
|
totalCorrect += s.correct;
|
|||
|
|
totalMs += s.totalMs;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ── 总汇总 ──
|
|||
|
|
console.log('\n============================================================');
|
|||
|
|
console.log(' 总汇总');
|
|||
|
|
console.log('============================================================');
|
|||
|
|
const totalPct = Math.round(totalCorrect / TEST_CASES.length * 100);
|
|||
|
|
const avgMs = Math.round(totalMs / TEST_CASES.length);
|
|||
|
|
console.log(` 总正确率: ${totalCorrect}/${TEST_CASES.length} (${totalPct}%)`);
|
|||
|
|
console.log(` 平均延迟: ${avgMs}ms`);
|
|||
|
|
|
|||
|
|
// ── 异常用例列表 ──
|
|||
|
|
const issues = results.filter(r => !r.matchExpect);
|
|||
|
|
if (issues.length > 0) {
|
|||
|
|
console.log(`\n ⚠️ 异常用例 (${issues.length}个):`);
|
|||
|
|
issues.forEach(r => {
|
|||
|
|
console.log(` - 【${r.cat}】${r.name}: ${r.hit ? 'HIT' : 'NO-HIT'} (期望${r.expectHit ? 'HIT' : 'NO-HIT'})`);
|
|||
|
|
});
|
|||
|
|
} else {
|
|||
|
|
console.log('\n 🎉 全部用例符合预期！');
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
runTest().catch(err => {
|
|||
|
|
console.error('测试失败:', err.message);
|
|||
|
|
process.exit(1);
|
|||
|
|
});
|