feat: conversation long-term memory + fix source ENUM bug
- New: conversationSummarizer.js (LLM summary every 3 turns, loadBestSummary, persistFinalSummary) - db/index.js: conversation_summaries table, upsertConversationSummary, getSessionSummary - redisClient.js: setSummary/getSummary (TTL 2h) - nativeVoiceGateway.js: _turnCount tracking, trigger summarize, persist on close - realtimeDialogRouting.js: inject summary context, reduce history 5->3 rounds - Fix: messages source ENUM missing 'search_knowledge' causing chat DB writes to fail
This commit is contained in:
@@ -27,21 +27,27 @@ const {
|
||||
const ToolExecutor = require('./toolExecutor');
|
||||
const {
|
||||
DEFAULT_VOICE_ASSISTANT_PROFILE,
|
||||
DEFAULT_CONSULTANT_CONTACT,
|
||||
resolveAssistantProfile,
|
||||
getAssistantDisplayName,
|
||||
buildVoiceSystemRole,
|
||||
buildVoiceGreeting,
|
||||
} = require('./assistantProfileConfig');
|
||||
const { getAssistantProfile } = require('./assistantProfileService');
|
||||
const redisClient = require('./redisClient');
|
||||
const { checkProductLinkTrigger } = require('./productLinkTrigger');
|
||||
const { triggerSummarizeIfNeeded, persistFinalSummary } = require('./conversationSummarizer');
|
||||
|
||||
const sessions = new Map();
|
||||
|
||||
const CONSULTANT_REFERRAL_PATTERN = /咨询(?:专业|你的)?顾问|健康管理顾问|联系顾问|一对一指导|咨询专业|咨询医生|咨询营养师|咨询专业人士|建议.*咨询|问问医生|问问.*营养师/;
|
||||
|
||||
const IDLE_TIMEOUT_MS = 5 * 60 * 1000;
|
||||
const AUDIO_KEEPALIVE_INTERVAL_MS = 20 * 1000;
|
||||
// 3200 bytes ≈ 66ms of silence at 24kHz s16le mono (larger frame to ensure S2S acceptance)
|
||||
const SILENT_AUDIO_FRAME = Buffer.alloc(3200, 0);
|
||||
|
||||
const DEFAULT_VOICE_BOT_NAME = DEFAULT_VOICE_ASSISTANT_PROFILE.nickname;
|
||||
const DEFAULT_VOICE_BOT_NAME = getAssistantDisplayName(DEFAULT_VOICE_ASSISTANT_PROFILE);
|
||||
|
||||
const DEFAULT_VOICE_SYSTEM_ROLE = buildVoiceSystemRole();
|
||||
|
||||
@@ -187,6 +193,7 @@ function persistUserSpeech(session, text) {
|
||||
session.lastPersistedUserAt = now;
|
||||
session.latestUserText = cleanText;
|
||||
session.latestUserTurnSeq = (session.latestUserTurnSeq || 0) + 1;
|
||||
session._turnCount = (session._turnCount || 0) + 1;
|
||||
resetIdleTimer(session);
|
||||
db.addMessage(session.sessionId, 'user', cleanText, 'voice_asr').catch((e) => console.warn('[NativeVoice][DB] add user failed:', e.message));
|
||||
redisClient.pushMessage(session.sessionId, { role: 'user', content: cleanText, source: 'voice_asr' }).catch(() => {});
|
||||
@@ -223,6 +230,24 @@ function persistAssistantSpeech(session, text, { source = 'voice_bot', toolName
|
||||
toolName,
|
||||
sequence: `native_assistant_${now}`,
|
||||
});
|
||||
// 异步触发摘要检查(每N轮)
|
||||
if (persistToDb) {
|
||||
triggerSummarizeIfNeeded(session, session.sessionId);
|
||||
}
|
||||
if (CONSULTANT_REFERRAL_PATTERN.test(cleanText)) {
|
||||
const contact = session.consultantContact || DEFAULT_CONSULTANT_CONTACT;
|
||||
if (contact.mobile || contact.wx_qr_code || contact.wechat_id) {
|
||||
console.log(`[NativeVoice] consultant referral detected session=${session.sessionId} text=${JSON.stringify(cleanText.slice(0, 80))}`);
|
||||
sendJson(session.client, {
|
||||
type: 'consultant_contact',
|
||||
name: contact.name || '大沃专业健康管理顾问',
|
||||
mobile: contact.mobile || '',
|
||||
wx_qr_code: contact.wx_qr_code || '',
|
||||
wechat_id: contact.wechat_id || '',
|
||||
message: '如需个性化健康建议,可联系大沃专业健康管理顾问',
|
||||
});
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -250,6 +275,7 @@ function flushAssistantStream(session, { source = 'voice_bot', toolName = null,
|
||||
return persistAssistantSpeech(session, fullText, { source, toolName, meta });
|
||||
}
|
||||
|
||||
|
||||
async function loadHandoffSummaryForVoice(session) {
|
||||
try {
|
||||
const history = await db.getHistoryForLLM(session.sessionId, 10);
|
||||
@@ -404,6 +430,7 @@ function clearUpstreamSuppression(session) {
|
||||
session.pendingAssistantSource = null;
|
||||
session.pendingAssistantToolName = null;
|
||||
session.pendingAssistantMeta = null;
|
||||
session._pendingEvidencePack = null;
|
||||
session.pendingAssistantTurnSeq = 0;
|
||||
session.blockUpstreamAudio = false;
|
||||
sendJson(session.client, { type: 'assistant_pending', active: false });
|
||||
@@ -487,7 +514,18 @@ async function processReply(session, text, turnSeq = session.latestUserTurnSeq |
|
||||
if (!resolveResult) {
|
||||
resolveResult = await resolveReply(session.sessionId, session, cleanText);
|
||||
}
|
||||
const { delivery, speechText, ragItems, source, toolName, routeDecision, responseMeta } = resolveResult;
|
||||
const { delivery, speechText, ragItems, source, toolName, routeDecision, responseMeta, evidencePack } = resolveResult;
|
||||
// 产品链接触发检测:用户请求查看产品详情时推送对应链接
|
||||
const productLinkResult = checkProductLinkTrigger(cleanText);
|
||||
if (productLinkResult.triggered && productLinkResult.product) {
|
||||
console.log(`[NativeVoice] product link triggered session=${session.sessionId} product=${productLinkResult.product.name}`);
|
||||
sendJson(session.client, {
|
||||
type: 'product_link',
|
||||
product: productLinkResult.product.name,
|
||||
link: productLinkResult.product.link,
|
||||
description: productLinkResult.product.description,
|
||||
});
|
||||
}
|
||||
if (activeTurnSeq !== (session.latestUserTurnSeq || 0)) {
|
||||
console.log(`[NativeVoice] stale reply ignored session=${session.sessionId} activeTurn=${activeTurnSeq} latestTurn=${session.latestUserTurnSeq || 0}`);
|
||||
clearUpstreamSuppression(session);
|
||||
@@ -501,6 +539,7 @@ async function processReply(session, text, turnSeq = session.latestUserTurnSeq |
|
||||
}
|
||||
session.blockUpstreamAudio = false;
|
||||
session._lastPartialAt = 0;
|
||||
session._pendingEvidencePack = null;
|
||||
session.awaitingUpstreamReply = true;
|
||||
session.pendingAssistantSource = 'voice_bot';
|
||||
session.pendingAssistantToolName = null;
|
||||
@@ -515,7 +554,7 @@ async function processReply(session, text, turnSeq = session.latestUserTurnSeq |
|
||||
}
|
||||
session.discardNextAssistantResponse = true;
|
||||
sendJson(session.client, { type: 'tts_reset', reason: 'knowledge_hit' });
|
||||
const ragContent = (ragItems || []).filter((item) => item && item.content);
|
||||
const ragContent = (ragItems || []).filter((item) => item && item.content && item.kind !== 'context');
|
||||
if (ragContent.length > 0) {
|
||||
console.log(`[NativeVoice] processReply sending external_rag to S2S session=${session.sessionId} route=${routeDecision?.route || 'unknown'} items=${ragContent.length}`);
|
||||
// KB话题记忆:记录本轮用户原始问题和时间戳,用于保护窗口和追问enrichment
|
||||
@@ -523,6 +562,7 @@ async function processReply(session, text, turnSeq = session.latestUserTurnSeq |
|
||||
session._lastKbTopic = cleanText;
|
||||
session._lastKbHitAt = Date.now();
|
||||
}
|
||||
session._pendingEvidencePack = evidencePack || null;
|
||||
// 不提前发KB原文作字幕:等S2S event 351返回实际语音文本后再更新字幕
|
||||
// 这样字幕和语音保持一致(S2S会基于RAG内容生成自然口语化的回答)
|
||||
session._pendingExternalRagReply = true;
|
||||
@@ -708,6 +748,7 @@ function handleUpstreamMessage(session, data) {
|
||||
session.pendingAssistantSource = null;
|
||||
session.pendingAssistantToolName = null;
|
||||
session.pendingAssistantMeta = null;
|
||||
session._pendingEvidencePack = null;
|
||||
console.log(`[NativeVoice] duplicate assistant final ignored (351) session=${session.sessionId} turn=${pendingAssistantTurnSeq}`);
|
||||
return;
|
||||
}
|
||||
@@ -725,30 +766,17 @@ function handleUpstreamMessage(session, data) {
|
||||
if (session._pendingExternalRagReply) {
|
||||
session._pendingExternalRagReply = false;
|
||||
}
|
||||
// 品牌安全检测:最终助手文本包含有害内容时,阻断音频并注入安全回复
|
||||
if (isBrandHarmful(assistantText)) {
|
||||
console.warn(`[NativeVoice][SafeGuard] harmful content in final assistant text, blocking session=${session.sessionId} text=${JSON.stringify(assistantText.slice(0, 120))}`);
|
||||
console.log(`[NativeVoice] upstream assistant session=${session.sessionId} text=${JSON.stringify(assistantText.slice(0, 120))}`);
|
||||
session.lastDeliveredAssistantTurnSeq = pendingAssistantTurnSeq;
|
||||
persistAssistantSpeech(session, assistantText, {
|
||||
source: pendingAssistantSource,
|
||||
toolName: pendingAssistantToolName,
|
||||
meta: pendingAssistantMeta,
|
||||
});
|
||||
// KB回复完成后重新阻断音频,防止下一个问题的S2S默认回复在early block前泄露
|
||||
if (session.currentTtsType === 'external_rag') {
|
||||
session.blockUpstreamAudio = true;
|
||||
sendJson(session.client, { type: 'tts_reset', reason: 'harmful_blocked' });
|
||||
const safeReply = getVoiceSafeReply();
|
||||
session.lastDeliveredAssistantTurnSeq = pendingAssistantTurnSeq;
|
||||
persistAssistantSpeech(session, safeReply, { source: 'voice_bot' });
|
||||
sendSpeechText(session, safeReply).catch((err) => {
|
||||
console.warn('[NativeVoice][SafeGuard] sendSpeechText failed:', err.message);
|
||||
});
|
||||
} else {
|
||||
console.log(`[NativeVoice] upstream assistant session=${session.sessionId} text=${JSON.stringify(assistantText.slice(0, 120))}`);
|
||||
session.lastDeliveredAssistantTurnSeq = pendingAssistantTurnSeq;
|
||||
persistAssistantSpeech(session, assistantText, {
|
||||
source: pendingAssistantSource,
|
||||
toolName: pendingAssistantToolName,
|
||||
meta: pendingAssistantMeta,
|
||||
});
|
||||
// KB回复完成后重新阻断音频,防止下一个问题的S2S默认回复在early block前泄露
|
||||
if (session.currentTtsType === 'external_rag') {
|
||||
session.blockUpstreamAudio = true;
|
||||
console.log(`[NativeVoice] re-blocked after KB response session=${session.sessionId}`);
|
||||
}
|
||||
console.log(`[NativeVoice] re-blocked after KB response session=${session.sessionId}`);
|
||||
}
|
||||
} else {
|
||||
const didFlush = flushAssistantStream(session, {
|
||||
@@ -763,6 +791,7 @@ function handleUpstreamMessage(session, data) {
|
||||
session.pendingAssistantSource = null;
|
||||
session.pendingAssistantToolName = null;
|
||||
session.pendingAssistantMeta = null;
|
||||
session._pendingEvidencePack = null;
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -845,6 +874,7 @@ function handleUpstreamMessage(session, data) {
|
||||
session.pendingAssistantSource = null;
|
||||
session.pendingAssistantToolName = null;
|
||||
session.pendingAssistantMeta = null;
|
||||
session._pendingEvidencePack = null;
|
||||
console.log(`[NativeVoice] duplicate assistant final ignored (559) session=${session.sessionId} turn=${pendingAssistantTurnSeq}`);
|
||||
return;
|
||||
}
|
||||
@@ -862,6 +892,7 @@ function handleUpstreamMessage(session, data) {
|
||||
session.pendingAssistantSource = null;
|
||||
session.pendingAssistantToolName = null;
|
||||
session.pendingAssistantMeta = null;
|
||||
session._pendingEvidencePack = null;
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1042,7 +1073,7 @@ function attachClientHandlers(session) {
|
||||
...((parsed.assistantProfile && typeof parsed.assistantProfile === 'object') ? parsed.assistantProfile : {}),
|
||||
});
|
||||
session.assistantProfile = assistantProfile;
|
||||
session.botName = parsed.botName || assistantProfile.nickname || DEFAULT_VOICE_BOT_NAME;
|
||||
session.botName = parsed.botName || getAssistantDisplayName(assistantProfile) || DEFAULT_VOICE_BOT_NAME;
|
||||
session.systemRole = buildVoiceSystemRole(assistantProfile);
|
||||
session.speakingStyle = parsed.speakingStyle || session.speakingStyle || DEFAULT_VOICE_SPEAKING_STYLE;
|
||||
session.speaker = parsed.speaker || process.env.VOLC_S2S_SPEAKER_ID || 'zh_female_vv_jupiter_bigtts';
|
||||
@@ -1089,6 +1120,10 @@ function attachClientHandlers(session) {
|
||||
if (session.upstream && session.upstream.readyState === WebSocket.OPEN) {
|
||||
session.upstream.close();
|
||||
}
|
||||
// 会话结束时持久化摘要到 MySQL
|
||||
persistFinalSummary(session).catch((err) => {
|
||||
console.warn('[NativeVoice] persistFinalSummary failed:', err.message);
|
||||
});
|
||||
sessions.delete(session.sessionId);
|
||||
});
|
||||
}
|
||||
@@ -1164,6 +1199,7 @@ function createSession(client, sessionId) {
|
||||
_echoLogOnce: false,
|
||||
_fillerActive: false,
|
||||
_pendingExternalRagReply: false,
|
||||
_pendingEvidencePack: null,
|
||||
_lastPartialAt: 0,
|
||||
pendingKbPrequery: null,
|
||||
_kbPrequeryText: '',
|
||||
@@ -1171,7 +1207,7 @@ function createSession(client, sessionId) {
|
||||
_lastKbTopic: '',
|
||||
_lastKbHitAt: 0,
|
||||
assistantProfile,
|
||||
botName: assistantProfile.nickname,
|
||||
botName: getAssistantDisplayName(assistantProfile),
|
||||
systemRole: buildVoiceSystemRole(assistantProfile),
|
||||
speakingStyle: DEFAULT_VOICE_SPEAKING_STYLE,
|
||||
speaker: process.env.VOLC_S2S_SPEAKER_ID || 'zh_female_vv_jupiter_bigtts',
|
||||
@@ -1201,6 +1237,7 @@ function createSession(client, sessionId) {
|
||||
_lastFinalNormalized: '',
|
||||
_lastFinalAt: 0,
|
||||
_audioKeepaliveTimer: null,
|
||||
_turnCount: 0,
|
||||
};
|
||||
sessions.set(sessionId, session);
|
||||
attachClientHandlers(session);
|
||||
|
||||
Reference in New Issue
Block a user