feat: 添加realtime_dialog和realtime_dialog_external_rag_test项目,更新test2项目

This commit is contained in:
User
2026-03-13 13:06:46 +08:00
parent 9dab61345c
commit 5521b673f5
215 changed files with 7626 additions and 1876 deletions

View File

@@ -1,11 +1,133 @@
const express = require('express');
const router = express.Router();
const cozeChatService = require('../services/cozeChatService');
const arkChatService = require('../services/arkChatService');
const ToolExecutor = require('../services/toolExecutor');
const { shouldForceKnowledgeRoute } = require('../services/realtimeDialogRouting');
const db = require('../db');
// 存储文字对话的会话状态sessionId -> session
const chatSessions = new Map();
function normalizeAssistantText(text) {
return String(text || '')
.replace(/\r/g, ' ')
.replace(/\n{2,}/g, '。')
.replace(/\n/g, ' ')
.replace(/。{2,}/g, '。')
.replace(/([])\1+/g, '$1')
.replace(/([。!?;,])\s*([。!?;,])/g, '$2')
.replace(/\s+/g, ' ')
.trim();
}
async function loadHandoffMessages(sessionId, voiceSubtitles = []) {
let voiceMessages = [];
try {
const dbHistory = await db.getHistoryForLLM(sessionId, 20);
if (dbHistory.length > 0) {
voiceMessages = dbHistory;
console.log(`[Chat] Loaded ${dbHistory.length} messages from DB for session ${sessionId}`);
}
} catch (e) { console.warn('[DB] getHistoryForLLM failed:', e.message); }
if (voiceMessages.length === 0 && voiceSubtitles.length > 0) {
const recentSubtitles = voiceSubtitles.slice(-10);
for (const sub of recentSubtitles) {
voiceMessages.push({
role: sub.role === 'user' ? 'user' : 'assistant',
content: sub.text,
});
}
}
return voiceMessages;
}
async function buildChatSessionState(sessionId, voiceSubtitles = []) {
const voiceMessages = await loadHandoffMessages(sessionId, voiceSubtitles);
let handoffSummary = '';
try {
handoffSummary = await arkChatService.summarizeContextForHandoff(voiceMessages, 3);
} catch (error) {
console.warn('[Chat] summarizeContextForHandoff failed:', error.message);
}
return {
userId: `user_${sessionId.slice(0, 12)}`,
conversationId: null,
voiceMessages,
handoffSummary,
handoffSummaryUsed: false,
createdAt: Date.now(),
lastActiveAt: Date.now(),
fromVoice: voiceSubtitles.length > 0 || voiceMessages.length > 0,
};
}
function buildInitialContextMessages(session) {
const summary = String(session?.handoffSummary || '').trim();
const extraMessages = [];
if (summary && !session?.handoffSummaryUsed) {
extraMessages.push({ role: 'assistant', content: `会话交接摘要:${summary}` });
}
if (Array.isArray(session?.voiceMessages) && session.voiceMessages.length > 0) {
extraMessages.push(...session.voiceMessages.slice(-6));
}
return extraMessages;
}
async function buildKnowledgeContextMessages(sessionId, session) {
const dbHistory = await db.getHistoryForLLM(sessionId, 20).catch(() => []);
const summary = String(session?.handoffSummary || '').trim();
if (!summary || session?.handoffSummaryUsed) {
return dbHistory;
}
return [
{ role: 'assistant', content: `会话交接摘要:${summary}` },
...dbHistory,
];
}
function extractKnowledgeReply(result) {
if (result && result.results && Array.isArray(result.results)) {
return result.results.map((item) => item.content || JSON.stringify(item)).join('\n');
}
if (result && result.error) {
return result.error;
}
return typeof result === 'string' ? result : '';
}
async function tryKnowledgeReply(sessionId, session, message) {
const text = String(message || '').trim();
if (!text) return null;
const context = await buildKnowledgeContextMessages(sessionId, session);
if (!shouldForceKnowledgeRoute(text, context)) {
return null;
}
const result = await ToolExecutor.execute('search_knowledge', { query: text }, context);
const content = normalizeAssistantText(extractKnowledgeReply(result));
if (!content) {
return null;
}
return {
content,
meta: {
route: 'search_knowledge',
original_text: text,
tool_name: 'search_knowledge',
tool_args: { query: text },
source: result?.source || null,
original_query: result?.original_query || text,
rewritten_query: result?.rewritten_query || null,
hit: typeof result?.hit === 'boolean' ? result.hit : null,
reason: result?.reason || null,
error_type: result?.errorType || null,
latency_ms: result?.latency_ms || null,
},
};
}
/**
* POST /api/chat/start
* 创建文字对话会话,可选传入语音通话的历史字幕
@@ -21,46 +143,21 @@ router.post('/start', async (req, res) => {
return res.status(500).json({ success: false, error: 'Coze 智能体未配置,请设置 COZE_API_TOKEN 和 COZE_BOT_ID' });
}
// 优先从数据库加载完整历史(包含语音通话中的工具结果等)
let voiceMessages = [];
try {
const dbHistory = await db.getHistoryForLLM(sessionId, 20);
if (dbHistory.length > 0) {
voiceMessages = dbHistory;
console.log(`[Chat] Loaded ${dbHistory.length} messages from DB for session ${sessionId}`);
}
} catch (e) { console.warn('[DB] getHistoryForLLM failed:', e.message); }
// 如果数据库没有历史,回退到 voiceSubtitles
if (voiceMessages.length === 0 && voiceSubtitles.length > 0) {
const recentSubtitles = voiceSubtitles.slice(-10);
for (const sub of recentSubtitles) {
voiceMessages.push({
role: sub.role === 'user' ? 'user' : 'assistant',
content: sub.text,
});
}
}
const sessionState = await buildChatSessionState(sessionId, voiceSubtitles);
// 更新数据库会话模式为 chat
try { await db.createSession(sessionId, `user_${sessionId.slice(0, 12)}`, 'chat'); } catch (e) {}
chatSessions.set(sessionId, {
userId: `user_${sessionId.slice(0, 12)}`,
conversationId: null,
voiceMessages,
createdAt: Date.now(),
fromVoice: voiceSubtitles.length > 0 || voiceMessages.length > 0,
});
chatSessions.set(sessionId, sessionState);
console.log(`[Chat] Session started: ${sessionId}, fromVoice: ${voiceSubtitles.length > 0}, voiceMessages: ${voiceMessages.length}`);
console.log(`[Chat] Session started: ${sessionId}, fromVoice: ${sessionState.fromVoice}, voiceMessages: ${sessionState.voiceMessages.length}, summary: ${sessionState.handoffSummary ? 'yes' : 'no'}`);
res.json({
success: true,
data: {
sessionId,
messageCount: voiceMessages.length,
fromVoice: voiceSubtitles.length > 0 || voiceMessages.length > 0,
messageCount: sessionState.voiceMessages.length,
fromVoice: sessionState.fromVoice,
},
});
});
@@ -81,23 +178,30 @@ router.post('/send', async (req, res) => {
// 自动创建会话(如果不存在)
if (!session) {
session = {
userId: `user_${sessionId.slice(0, 12)}`,
conversationId: null,
voiceMessages: [],
createdAt: Date.now(),
fromVoice: false,
};
session = await buildChatSessionState(sessionId, []);
chatSessions.set(sessionId, session);
}
session.lastActiveAt = Date.now();
console.log(`[Chat] User(${sessionId}): ${message}`);
// 写入数据库:用户消息
db.addMessage(sessionId, 'user', message, 'chat_user').catch(e => console.warn('[DB] addMessage failed:', e.message));
const knowledgeReply = await tryKnowledgeReply(sessionId, session, message);
if (knowledgeReply) {
session.handoffSummaryUsed = true;
db.addMessage(sessionId, 'assistant', knowledgeReply.content, 'chat_bot', 'search_knowledge', knowledgeReply.meta).catch(e => console.warn('[DB] addMessage failed:', e.message));
return res.json({
success: true,
data: {
content: knowledgeReply.content,
},
});
}
// 首次对话时注入语音历史作为上下文,之后 Coze 自动管理会话历史
const extraMessages = !session.conversationId ? session.voiceMessages : [];
const extraMessages = !session.conversationId ? buildInitialContextMessages(session) : [];
const result = await cozeChatService.chat(
session.userId,
@@ -105,21 +209,23 @@ router.post('/send', async (req, res) => {
session.conversationId,
extraMessages
);
const normalizedContent = normalizeAssistantText(result.content);
// 保存 Coze 返回的 conversationId
session.conversationId = result.conversationId;
session.handoffSummaryUsed = true;
console.log(`[Chat] Assistant(${sessionId}): ${result.content?.substring(0, 100)}`);
console.log(`[Chat] Assistant(${sessionId}): ${normalizedContent?.substring(0, 100)}`);
// 写入数据库AI 回复
if (result.content) {
db.addMessage(sessionId, 'assistant', result.content, 'chat_bot').catch(e => console.warn('[DB] addMessage failed:', e.message));
if (normalizedContent) {
db.addMessage(sessionId, 'assistant', normalizedContent, 'chat_bot').catch(e => console.warn('[DB] addMessage failed:', e.message));
}
res.json({
success: true,
data: {
content: result.content,
content: normalizedContent,
},
});
} catch (error) {
@@ -160,15 +266,10 @@ router.post('/send-stream', async (req, res) => {
let session = chatSessions.get(sessionId);
if (!session) {
session = {
userId: `user_${sessionId.slice(0, 12)}`,
conversationId: null,
voiceMessages: [],
createdAt: Date.now(),
fromVoice: false,
};
session = await buildChatSessionState(sessionId, []);
chatSessions.set(sessionId, session);
}
session.lastActiveAt = Date.now();
console.log(`[Chat][SSE] User(${sessionId}): ${message}`);
@@ -182,9 +283,17 @@ router.post('/send-stream', async (req, res) => {
res.setHeader('X-Accel-Buffering', 'no');
res.flushHeaders();
const knowledgeReply = await tryKnowledgeReply(sessionId, session, message);
if (knowledgeReply) {
session.handoffSummaryUsed = true;
db.addMessage(sessionId, 'assistant', knowledgeReply.content, 'chat_bot', 'search_knowledge', knowledgeReply.meta).catch(e => console.warn('[DB] addMessage failed:', e.message));
res.write(`data: ${JSON.stringify({ type: 'done', content: knowledgeReply.content })}\n\n`);
return res.end();
}
try {
// 首次对话时注入语音历史作为上下文
const extraMessages = !session.conversationId ? session.voiceMessages : [];
const extraMessages = !session.conversationId ? buildInitialContextMessages(session) : [];
const result = await cozeChatService.chatStream(
session.userId,
@@ -198,17 +307,19 @@ router.post('/send-stream', async (req, res) => {
onDone: () => {},
}
);
const normalizedContent = normalizeAssistantText(result.content);
// 保存 Coze 返回的 conversationId
session.conversationId = result.conversationId;
console.log(`[Chat][SSE] Assistant(${sessionId}): ${result.content?.substring(0, 100)}`);
session.handoffSummaryUsed = true;
console.log(`[Chat][SSE] Assistant(${sessionId}): ${normalizedContent?.substring(0, 100)}`);
// 写入数据库AI 回复
if (result.content) {
db.addMessage(sessionId, 'assistant', result.content, 'chat_bot').catch(e => console.warn('[DB] addMessage failed:', e.message));
if (normalizedContent) {
db.addMessage(sessionId, 'assistant', normalizedContent, 'chat_bot').catch(e => console.warn('[DB] addMessage failed:', e.message));
}
res.write(`data: ${JSON.stringify({ type: 'done', content: result.content })}\n\n`);
res.write(`data: ${JSON.stringify({ type: 'done', content: normalizedContent })}\n\n`);
res.end();
} catch (error) {
console.error('[Chat][SSE] Stream failed:', error.message);
@@ -231,7 +342,7 @@ setInterval(() => {
const now = Date.now();
const TTL = 30 * 60 * 1000;
for (const [id, session] of chatSessions) {
if (now - session.createdAt > TTL) {
if (now - (session.lastActiveAt || session.createdAt) > TTL) {
chatSessions.delete(id);
console.log(`[Chat] Session expired and cleaned: ${id}`);
}

View File

@@ -2,6 +2,48 @@ const express = require('express');
const router = express.Router();
const db = require('../db');
/**
* GET /api/session/list
* 获取会话列表(按更新时间倒序,带最后一条消息预览)
*/
router.get('/list', async (req, res) => {
try {
const userId = req.query.userId || null;
const limit = parseInt(req.query.limit) || 50;
const sessions = await db.getSessionList(userId, limit);
res.json({
success: true,
data: sessions.map((s) => ({
id: s.id,
userId: s.user_id,
mode: s.mode,
createdAt: s.created_at,
updatedAt: s.updated_at,
lastMessage: s.last_message ? (s.last_message.length > 60 ? s.last_message.slice(0, 60) + '...' : s.last_message) : null,
messageCount: parseInt(s.message_count) || 0,
})),
});
} catch (err) {
console.error('[Session] List failed:', err.message);
res.status(500).json({ success: false, error: err.message });
}
});
/**
* DELETE /api/session/:id
* 删除会话及其所有消息
*/
router.delete('/:id', async (req, res) => {
try {
const { id } = req.params;
await db.deleteSession(id);
res.json({ success: true });
} catch (err) {
console.error('[Session] Delete failed:', err.message);
res.status(500).json({ success: false, error: err.message });
}
});
/**
* GET /api/session/:id/history
* 获取会话完整历史(用于文字↔语音切换时加载上下文)

View File

@@ -1,20 +1,11 @@
const express = require('express');
const router = express.Router();
const { v4: uuidv4 } = require('uuid');
const volcengine = require('../services/volcengine');
const VoiceChatConfigBuilder = require('../config/voiceChatConfig');
const ToolExecutor = require('../services/toolExecutor');
const DEFAULT_TOOLS = require('../config/tools');
const db = require('../db');
const activeSessions = new Map();
const completedSessions = new Map();
const roomToBotUserId = new Map();
const roomToHumanUserId = new Map();
const roomToSessionId = new Map();
const roomToTaskId = new Map();
const latestUserSpeech = new Map();
const toolCallBuffers = new Map();
const directSessions = new Map();
router.get('/config', (req, res) => {
res.json({
@@ -44,516 +35,133 @@ router.get('/config', (req, res) => {
});
});
router.post('/prepare', async (req, res) => {
router.post('/direct/session', async (req, res) => {
try {
const { userId } = req.body;
if (!userId) {
return res.status(400).json({ success: false, error: 'userId is required' });
}
const sessionId = uuidv4();
const roomId = `room_${sessionId.slice(0, 8)}`;
const taskId = `task_${sessionId.slice(0, 8)}_${Date.now()}`;
const rtcToken = volcengine.generateRTCToken(roomId, userId);
activeSessions.set(sessionId, {
roomId,
taskId,
userId,
const { userId, sessionId } = req.body || {};
const sid = sessionId || uuidv4();
const directSession = {
sessionId: sid,
userId: userId || null,
startTime: Date.now(),
subtitles: [],
started: false,
direct: true,
};
directSessions.set(sid, directSession);
await db.createSession(sid, userId || null, 'voice');
res.json({
success: true,
data: {
sessionId: sid,
userId: userId || null,
},
});
roomToTaskId.set(roomId, taskId);
roomToSessionId.set(roomId, sessionId);
console.log(`[Voice] Session prepared: ${sessionId}, room: ${roomId}, user: ${userId}`);
try { await db.createSession(sessionId, userId, 'voice'); } catch (e) { console.warn('[DB] createSession failed:', e.message); }
} catch (error) {
console.error('[DirectVoice] Create session failed:', error.message);
res.status(500).json({ success: false, error: error.message });
}
});
router.post('/direct/message', async (req, res) => {
try {
const { sessionId, role, text, source, toolName } = req.body || {};
if (!sessionId || !text || !source) {
return res.status(400).json({ success: false, error: 'sessionId, text and source are required' });
}
await db.addMessage(sessionId, role === 'user' ? 'user' : 'assistant', text, source, toolName || null);
res.json({ success: true });
} catch (error) {
console.error('[DirectVoice] Add message failed:', error.message);
res.status(500).json({ success: false, error: error.message });
}
});
router.post('/diag', (req, res) => {
try {
const { sessionId, roomId, type, payload } = req.body || {};
console.log(`[Diag] type=${type || 'unknown'} session=${sessionId || '-'} room=${roomId || '-'} payload=${JSON.stringify(payload || {})}`);
res.json({ success: true });
} catch (error) {
console.error('[Diag] Error:', error.message);
res.status(500).json({ success: false, error: error.message });
}
});
router.post('/direct/query', async (req, res) => {
try {
const { sessionId, query, appendUserMessage } = req.body || {};
if (!sessionId) {
return res.status(400).json({ success: false, error: 'sessionId is required' });
}
const context = await db.getHistoryForLLM(sessionId, 20).catch(() => []);
const cleanQuery = (query || '').trim();
if (appendUserMessage && cleanQuery) {
await db.addMessage(sessionId, 'user', cleanQuery, 'voice_asr').catch(() => null);
}
const result = await ToolExecutor.execute('search_knowledge', { query: cleanQuery }, context);
let contentText = JSON.stringify(result);
if (result && result.results && Array.isArray(result.results)) {
contentText = result.results.map((item) => item.content || JSON.stringify(item)).join('\n');
} else if (result && result.error) {
contentText = result.error;
} else if (typeof result === 'string') {
contentText = result;
}
const ragItems = result && result.results && Array.isArray(result.results) && result.results.length > 0
? result.results.map((item) => ({
title: item.title || '知识库结果',
content: item.content || JSON.stringify(item),
}))
: [{
title: '知识库结果',
content: contentText,
}];
await db.addMessage(sessionId, 'assistant', contentText, 'voice_tool', 'search_knowledge', {
route: 'search_knowledge',
original_text: cleanQuery,
tool_name: 'search_knowledge',
tool_args: { query: cleanQuery },
source: result?.source || null,
original_query: result?.original_query || cleanQuery,
rewritten_query: result?.rewritten_query || null,
hit: typeof result?.hit === 'boolean' ? result.hit : null,
reason: result?.reason || null,
error_type: result?.errorType || null,
latency_ms: result?.latency_ms || null,
}).catch(() => null);
res.json({
success: true,
data: {
sessionId,
roomId,
taskId,
rtcToken,
rtcAppId: process.env.VOLC_RTC_APP_ID,
query: cleanQuery,
contentText,
ragItems,
ragJson: JSON.stringify(ragItems),
},
});
} catch (error) {
console.error('[Voice] Prepare failed:', error.message);
console.error('[DirectVoice] Query failed:', error.message);
res.status(500).json({ success: false, error: error.message });
}
});
router.post('/start', async (req, res) => {
let session = null;
router.post('/direct/stop', async (req, res) => {
try {
const {
sessionId,
botName,
systemRole,
speakingStyle,
modelVersion,
speaker,
enableWebSearch,
chatHistory,
} = req.body;
const { sessionId } = req.body || {};
if (!sessionId) {
return res.status(400).json({ success: false, error: 'sessionId is required' });
}
session = activeSessions.get(sessionId);
if (!session) {
return res.status(404).json({ success: false, error: 'Session not found' });
}
if (session.started) {
return res.json({ success: true, data: { message: 'Already started' } });
}
let effectiveChatHistory = chatHistory;
if ((!chatHistory || chatHistory.length === 0) && sessionId) {
try {
const dbHistory = await db.getHistoryForLLM(sessionId, 20);
if (dbHistory.length > 0) {
effectiveChatHistory = dbHistory;
console.log(`[Voice] Loaded ${dbHistory.length} messages from DB for session ${sessionId}`);
}
} catch (e) { console.warn('[DB] getHistoryForLLM failed:', e.message); }
}
console.log(`[Voice] chatHistory: ${effectiveChatHistory ? effectiveChatHistory.length : 'undefined'} messages`);
const { config, botUserId } = VoiceChatConfigBuilder.build({
roomId: session.roomId,
taskId: session.taskId,
userId: session.userId,
botName,
systemRole,
speakingStyle,
modelVersion,
speaker,
tools: DEFAULT_TOOLS,
enableWebSearch,
chatHistory: effectiveChatHistory,
});
session.botUserId = botUserId;
roomToBotUserId.set(session.roomId, botUserId);
roomToHumanUserId.set(session.roomId, session.userId);
console.log(`[Voice] room=${session.roomId} botUserId=${botUserId} humanUserId=${session.userId}`);
const result = await volcengine.startVoiceChat(config);
session.started = true;
// 捕获服务端可能分配的不同 TaskId
const serverTaskId = result?.Result?.TaskId || result?.Result?.task_id;
if (serverTaskId && serverTaskId !== session.taskId) {
console.log(`[Voice] Server assigned different TaskId: ${serverTaskId} (ours: ${session.taskId})`);
roomToTaskId.set(session.roomId, serverTaskId);
session.taskId = serverTaskId;
}
console.log(`[Voice] Session started: ${sessionId}, TaskId=${session.taskId}`);
directSessions.delete(sessionId);
const messages = await db.getMessages(sessionId).catch(() => []);
res.json({
success: true,
data: { startResult: result },
data: {
sessionId,
messageCount: messages.length,
},
});
} catch (error) {
const detail = error.response?.data || error.message;
console.error('[Voice] Start failed:', JSON.stringify(detail, null, 2));
if (session) {
try {
await volcengine.stopVoiceChat({
AppId: process.env.VOLC_RTC_APP_ID,
RoomId: session.roomId,
TaskId: session.taskId,
});
console.log(`[Voice] Stopped failed session`);
} catch (stopErr) {
console.warn('[Voice] Stop failed during error handling:', stopErr.message);
}
}
res.status(500).json({ success: false, error: typeof detail === 'string' ? detail : JSON.stringify(detail) });
}
});
router.post('/stop', async (req, res) => {
try {
const { sessionId } = req.body;
const session = activeSessions.get(sessionId);
if (session) {
await volcengine.stopVoiceChat({
AppId: process.env.VOLC_RTC_APP_ID,
RoomId: session.roomId,
TaskId: session.taskId,
});
const duration = Math.floor((Date.now() - session.startTime) / 1000);
console.log(`[Voice] Session stopped: ${sessionId}, duration: ${duration}s, subtitles: ${session.subtitles.length}`);
if (session.subtitles.length > 0) {
completedSessions.set(sessionId, {
subtitles: session.subtitles,
duration,
endTime: Date.now(),
});
setTimeout(() => completedSessions.delete(sessionId), 30 * 60 * 1000);
}
activeSessions.delete(sessionId);
roomToTaskId.delete(session.roomId);
roomToSessionId.delete(session.roomId);
roomToBotUserId.delete(session.roomId);
roomToHumanUserId.delete(session.roomId);
res.json({
success: true,
data: {
duration,
subtitleCount: session.subtitles.length,
subtitles: session.subtitles,
},
});
} else {
res.json({ success: true, data: { message: 'Session not found or already stopped' } });
}
} catch (error) {
console.error('[Voice] Stop failed:', error.message);
console.error('[DirectVoice] Stop session failed:', error.message);
res.status(500).json({ success: false, error: error.message });
}
});
router.post('/subtitle', (req, res) => {
try {
const { sessionId, roomId, text, role, definite, sequence } = req.body;
const session = activeSessions.get(sessionId);
if (definite && text) {
const subtitleRole = role === 'user' ? 'user' : 'assistant';
if (session) {
session.subtitles.push({ text, role: subtitleRole, timestamp: Date.now(), sequence });
}
const sid = sessionId || (session && roomToSessionId.get(session.roomId));
if (sid) {
const source = subtitleRole === 'user' ? 'voice_asr' : 'voice_bot';
db.addMessage(sid, subtitleRole, text, source).catch(e => console.warn('[DB] addMessage failed:', e.message));
}
if (subtitleRole === 'user') {
const rid = roomId || (session && session.roomId) || '';
if (rid) {
latestUserSpeech.set(rid, { text, timestamp: Date.now() });
console.log(`[Subtitle][user][${rid}] "${text}"`);
}
} else {
console.log(`[Subtitle][assistant] ${text}`);
}
}
res.json({ success: true });
} catch (error) {
console.error('[Subtitle] Error:', error.message);
res.status(500).json({ success: false, error: error.message });
}
});
router.get('/subtitles/:sessionId', (req, res) => {
const session = activeSessions.get(req.params.sessionId);
res.json({
success: true,
data: session ? session.subtitles : [],
});
});
function extractReadableText(chunks) {
const raw = chunks.join('');
let decoded = raw;
try {
decoded = decoded.replace(/\\\\u([0-9a-fA-F]{4})/g, (_, hex) => {
return String.fromCharCode(parseInt(hex, 16));
});
decoded = decoded.replace(/\\u([0-9a-fA-F]{4})/g, (_, hex) => {
return String.fromCharCode(parseInt(hex, 16));
});
} catch (e) { }
const chineseChars = decoded.match(/[\u4e00-\u9fff\u3400-\u4dbf]+/g) || [];
const skipWords = new Set(['id', 'type', 'function', 'name', 'arguments', 'query', 'object', 'string']);
const englishWords = (decoded.match(/[a-zA-Z]{2,}/g) || [])
.filter(w => !skipWords.has(w.toLowerCase()));
const parts = [...chineseChars, ...englishWords];
const result = parts.join(' ').trim();
console.log(`[FC] extractReadableText: chinese=[${chineseChars.join(',')}] english=[${englishWords.join(',')}] → "${result}"`);
return result;
}
let fcCallbackSeq = 0;
router.post('/fc_callback', async (req, res) => {
try {
const body = req.body;
if (!body || typeof body !== 'object' || Object.keys(body).length === 0) {
console.error('[FC] Empty body');
return res.status(400).json({ success: false, error: 'Empty body' });
}
const { Message, Signature, Type, RoomID, TaskID, TaskType, AppID, AppId, room_id, task_id, roomId, taskId } = body;
const effectiveRoomId = RoomID || room_id || roomId;
const effectiveTaskId = TaskID || task_id || taskId;
const effectiveAppId = AppID || AppId || process.env.VOLC_RTC_APP_ID;
const seq = body._seq || ++fcCallbackSeq;
console.log(`[FC] >>> Callback received: seq=${seq} Type="${Type}" Room=${effectiveRoomId} Task=${effectiveTaskId} TaskType=${TaskType}`);
let msgObj = null;
try {
msgObj = typeof Message === 'string' ? JSON.parse(Message) : Message;
} catch (e) {
console.error('[FC] Failed to parse Message:', e.message);
return res.json({ success: true });
}
if (Type === 'tool_calls' && Array.isArray(msgObj) && msgObj.length > 0) {
const tc = msgObj[0];
const chunkId = tc.id || '';
const chunkName = tc.function?.name || '';
const chunkArgs = tc.function?.arguments || '';
const existing = toolCallBuffers.get(effectiveTaskId);
if (existing && existing.triggered) {
const userSpeech = latestUserSpeech.get(effectiveRoomId);
const hasNewInput = userSpeech && (Date.now() - userSpeech.timestamp < 10000);
if (hasNewInput) {
console.log(`[FC] [FormatA] New user input detected, clearing cooldown for room=${effectiveRoomId}`);
toolCallBuffers.delete(effectiveTaskId);
} else {
// 扩展 cooldown 到 30 秒,防止 LLM 在 KB 查询期间无限重试
const cooldownMs = existing.resultSentAt ? 30000 : 15000;
const elapsed = existing.resultSentAt
? (Date.now() - existing.resultSentAt)
: (Date.now() - existing.createdAt);
if (elapsed < cooldownMs) {
console.log(`[FC] [FormatA] Cooldown active (${elapsed}ms < ${cooldownMs}ms), ignoring retry for TaskID=${effectiveTaskId}`);
res.json({ success: true });
return;
}
console.log(`[FC] [FormatA] Cooldown expired (${elapsed}ms >= ${cooldownMs}ms), allowing new call for TaskID=${effectiveTaskId}`);
toolCallBuffers.delete(effectiveTaskId);
}
}
if (!toolCallBuffers.has(effectiveTaskId)) {
toolCallBuffers.set(effectiveTaskId, {
id: '', name: '', chunks: [], triggered: false,
RoomID: effectiveRoomId, AppID: effectiveAppId, S2STaskID: effectiveTaskId, createdAt: Date.now(), timer: null,
});
console.log(`[FC] [FormatA] New buffer created for TaskID=${effectiveTaskId}, room=${effectiveRoomId}`);
}
const buf = toolCallBuffers.get(effectiveTaskId);
if (chunkId && !buf.id) buf.id = chunkId;
if (chunkName && !buf.name) buf.name = chunkName;
if (chunkArgs) {
buf.chunks.push({ seq: tc.seq || 0, args: chunkArgs });
}
res.json({ success: true });
if (buf.timer) clearTimeout(buf.timer);
buf.timer = setTimeout(async () => { // 500ms 收集 chunks
const b = toolCallBuffers.get(effectiveTaskId);
if (!b || b.triggered) return;
b.triggered = true;
const toolName = b.name || 'search_knowledge';
const sortedChunks = b.chunks.sort((a, b) => a.seq - b.seq);
const allArgs = sortedChunks.map(c => c.args).join('');
console.log(`[FC] [FormatA] 500ms timeout, ${b.chunks.length} chunks collected, name="${toolName}"`);
const s2sTaskId = roomToTaskId.get(b.RoomID) || b.S2STaskID || effectiveTaskId;
console.log(`[FC] TaskId resolution: roomToTaskId=${roomToTaskId.get(b.RoomID)} callback=${b.S2STaskID} → using=${s2sTaskId}`);
// 不再单独发 interrupt 命令ExternalTextToSpeech 的 InterruptMode:1 已包含打断功能
let parsedArgs = null;
try {
parsedArgs = JSON.parse(allArgs);
console.log(`[FC] [FormatA] JSON.parse succeeded: ${JSON.stringify(parsedArgs)}`);
} catch (e) {
const userSpeech = latestUserSpeech.get(b.RoomID);
if (userSpeech && (Date.now() - userSpeech.timestamp < 30000)) {
console.log(`[FC] [FormatA] Using ASR user speech: "${userSpeech.text}"`);
parsedArgs = { query: userSpeech.text };
} else {
const extractedText = extractReadableText(b.chunks.map(c => c.args));
console.log(`[FC] [FormatA] No ASR text, extracted from chunks: "${extractedText}"`);
parsedArgs = { query: extractedText || '' };
}
}
console.log('[FC] ⚡ Starting KB query (no pre-query interrupt)');
const kbResult = await ToolExecutor.execute(toolName, parsedArgs);
try {
const result = kbResult;
const resultStr = JSON.stringify(result);
console.log(`[FC] Tool result (${toolName}): ${resultStr.substring(0, 500)}`);
let contentText = resultStr;
try {
if (result && result.results && Array.isArray(result.results)) {
contentText = result.results.map(r => r.content || JSON.stringify(r)).join('\n');
} else if (result && result.error) {
contentText = result.error;
} else if (typeof result === 'string') {
contentText = result;
}
} catch (e) { }
const dbSessionId = roomToSessionId.get(b.RoomID);
if (dbSessionId) {
db.addMessage(dbSessionId, 'assistant', contentText, 'voice_tool', toolName)
.catch(e => console.warn('[DB] addMessage(tool) failed:', e.message));
}
console.log(`[FC] Knowledge base content (${contentText.length} chars): ${contentText.substring(0, 200)}${contentText.length > 200 ? '...' : ''}`);
b.resultSentAt = Date.now();
// === 策略:只用 Command:function 回传结果给 LLM ===
// 根因分析:
// 1. ExternalTextToSpeech 在 S2S 端到端模式下不产生可听见的音频API返回ok但无声音
// 2. ExternalTextToSpeech InterruptMode=1 会打断正在播放的 S2S 回复,导致用户听到中断
// 3. Command:function 是官方自定义 FC 模式的正确回传方式
// 流程Command:function → LLM 收到工具结果 → LLM 生成回复 → S2S 朗读
const toolCallId = b.id || 'unknown_call_id';
const functionContent = contentText.length > 1500
? contentText.substring(0, 1500) + '……(内容较长,以上为主要部分)'
: contentText;
const funcMsg = JSON.stringify({
ToolCallID: toolCallId,
Content: functionContent,
});
let activeTaskId = s2sTaskId;
try {
console.log(`[FC] ★ Sending Command:function (ToolCallID=${toolCallId}, content=${functionContent.length} chars)`);
await volcengine.updateVoiceChat({
AppId: effectiveAppId,
RoomId: b.RoomID,
TaskId: activeTaskId,
Command: 'function',
Message: funcMsg,
});
console.log('[FC] ✅ Command:function sent OK → LLM will generate S2S response with KB content');
} catch (funcErr) {
console.error('[FC] ✖ Command:function failed:', funcErr.message);
// 如果正式 TaskId 失败,尝试回调 TaskId
if (activeTaskId !== b.S2STaskID) {
try {
console.log(`[FC] Retrying Command:function with callback TaskID=${b.S2STaskID}`);
activeTaskId = b.S2STaskID;
await volcengine.updateVoiceChat({
AppId: effectiveAppId,
RoomId: b.RoomID,
TaskId: activeTaskId,
Command: 'function',
Message: funcMsg,
});
console.log('[FC] ✅ Command:function retry OK');
} catch (retryErr) {
console.error('[FC] ✖ Command:function retry also failed:', retryErr.message);
}
}
}
console.log(`[FC] Final result: Command:function sent (${functionContent.length} chars)`);
} catch (err) {
console.error(`[FC] Tool execution failed:`, err.message);
console.error(`[FC] Error details:`, err);
}
}, 500); // 从1s减到500ms减少等待
return;
}
if (msgObj && typeof msgObj === 'object' && !Array.isArray(msgObj)) {
const eventType = msgObj.event_type;
console.log(`[FC] [FormatB] event_type="${eventType}"`);
if (eventType === 'function_calling') {
const funcName = msgObj.function || '';
const toolCallId = msgObj.tool_call_id || '';
const responseId = msgObj.response_id || '';
console.log(`[FC] [Information] FC notification: func=${funcName} toolCallId=${toolCallId} responseId=${responseId}`);
res.json({ success: true });
// ExternalTextToSpeech 在 S2S 模式下不产生音频,不再发送安抚语
// LLM 的 tool_calls 会触发 FormatA 分支执行工具并通过 Command:function 回传结果
console.log(`[FC] [Information] FC notification received, waiting for tool_calls`);
return;
}
}
if (msgObj && typeof msgObj === 'object') {
const asrText = msgObj.text || msgObj.asr_text || msgObj.content ||
msgObj.user_text || msgObj.transcript ||
(msgObj.data && (msgObj.data.text || msgObj.data.asr_text || msgObj.data.content));
const role = msgObj.role || msgObj.speaker || msgObj.data?.role || '';
const isUser = !role || role === 'user' || role === 'human';
if (asrText && isUser && RoomID) {
latestUserSpeech.set(RoomID, { text: asrText, timestamp: Date.now() });
console.log(`[FC] [ConvState] Stored user speech for ${RoomID}: "${asrText}"`);
}
}
res.json({ success: true });
} catch (error) {
console.error('[FC] Error:', error.message);
res.status(500).json({ success: false, error: error.message });
}
});
router.post('/room_message', (req, res) => {
try {
const { roomId, uid, text } = req.body;
if (!roomId || !text) {
return res.json({ success: true });
}
const jsonStart = text.search(/[\[{]/);
if (jsonStart < 0) {
return res.json({ success: true });
}
const jsonStr = text.substring(jsonStart);
let parsed = null;
try { parsed = JSON.parse(jsonStr); } catch (e) {
const textMatch = jsonStr.match(/"text"\s*:\s*"([^"]+)"/);
if (textMatch && textMatch[1]) {
const extractedText = textMatch[1];
const userIdMatch = jsonStr.match(/"userId"\s*:\s*"([^"]+)"/);
const subtitleUserId = userIdMatch ? userIdMatch[1] : '';
const isUserSpeech = subtitleUserId && !subtitleUserId.startsWith('bot_');
if (isUserSpeech && extractedText) {
latestUserSpeech.set(roomId, { text: extractedText, timestamp: Date.now(), source: 'room_regex' });
console.log(`[RoomMsg] ✅ Stored user speech (regex) for ${roomId}: "${extractedText}"`);
}
}
return res.json({ success: true });
}
if (parsed && parsed.data && Array.isArray(parsed.data)) {
parsed.data.forEach(sub => {
const subText = sub.text || '';
const subUserId = sub.userId || sub.user_id || '';
const isDefinite = sub.definite === true;
const isUserSpeech = subUserId && !subUserId.startsWith('bot_');
if (subText && isUserSpeech && isDefinite) {
latestUserSpeech.set(roomId, { text: subText, timestamp: Date.now(), source: 'room_subtitle' });
console.log(`[RoomMsg] ✅ Stored user speech for ${roomId}: "${subText}"`);
}
});
res.json({ success: true });
return;
}
if (parsed && typeof parsed === 'object') {
const asrText = parsed.text || parsed.asr_text || parsed.content ||
parsed.user_text || parsed.transcript ||
(parsed.data && typeof parsed.data === 'string' ? parsed.data : null);
const isBot = uid && uid.startsWith('bot_');
if (asrText && !isBot) {
latestUserSpeech.set(roomId, { text: asrText, timestamp: Date.now(), source: 'room_object' });
console.log(`[RoomMsg] ✅ Stored user speech (obj) for ${roomId}: "${asrText}"`);
}
}
res.json({ success: true });
} catch (error) {
console.error('[RoomMsg] Error:', error.message);
res.json({ success: true });
}
});
router.post('/tool-callback', async (req, res) => {
console.log('[ToolCallback] Legacy callback received:', JSON.stringify(req.body));
res.json({ success: true, message: 'deprecated, use fc_callback instead' });
});
router.get('/sessions', (req, res) => {
const sessions = [];
for (const [id, session] of activeSessions) {
sessions.push({
sessionId: id,
roomId: session.roomId,
userId: session.userId,
duration: Math.floor((Date.now() - session.startTime) / 1000),
subtitleCount: session.subtitles.length,
});
}
res.json({ success: true, data: sessions });
});
module.exports = router;